xref: /XiangShan/src/main/scala/xiangshan/frontend/NewFtq.scala (revision 3f88c020b1ffa2f2164b131a12f590e2530f2e23)
1/***************************************************************************************
2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
3* Copyright (c) 2020-2021 Peng Cheng Laboratory
4*
5* XiangShan is licensed under Mulan PSL v2.
6* You can use this software according to the terms and conditions of the Mulan PSL v2.
7* You may obtain a copy of Mulan PSL v2 at:
8*          http://license.coscl.org.cn/MulanPSL2
9*
10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
13*
14* See the Mulan PSL v2 for more details.
15***************************************************************************************/
16
17package xiangshan.frontend
18
19import chipsalliance.rocketchip.config.Parameters
20import chisel3._
21import chisel3.util._
22import utils._
23import utility._
24import xiangshan._
25import xiangshan.frontend.icache._
26import xiangshan.backend.CtrlToFtqIO
27import xiangshan.backend.decode.ImmUnion
28import utility.ChiselDB
29
30class FtqDebugBundle extends Bundle {
31  val pc = UInt(39.W)
32  val target = UInt(39.W)
33  val isBr = Bool()
34  val isJmp = Bool()
35  val isCall = Bool()
36  val isRet = Bool()
37  val misPred = Bool()
38  val isTaken = Bool()
39  val predStage = UInt(2.W)
40}
41
42class FtqPtr(implicit p: Parameters) extends CircularQueuePtr[FtqPtr](
43  p => p(XSCoreParamsKey).FtqSize
44){
45}
46
47object FtqPtr {
48  def apply(f: Bool, v: UInt)(implicit p: Parameters): FtqPtr = {
49    val ptr = Wire(new FtqPtr)
50    ptr.flag := f
51    ptr.value := v
52    ptr
53  }
54  def inverse(ptr: FtqPtr)(implicit p: Parameters): FtqPtr = {
55    apply(!ptr.flag, ptr.value)
56  }
57}
58
59class FtqNRSRAM[T <: Data](gen: T, numRead: Int)(implicit p: Parameters) extends XSModule {
60
61  val io = IO(new Bundle() {
62    val raddr = Input(Vec(numRead, UInt(log2Up(FtqSize).W)))
63    val ren = Input(Vec(numRead, Bool()))
64    val rdata = Output(Vec(numRead, gen))
65    val waddr = Input(UInt(log2Up(FtqSize).W))
66    val wen = Input(Bool())
67    val wdata = Input(gen)
68  })
69
70  for(i <- 0 until numRead){
71    val sram = Module(new SRAMTemplate(gen, FtqSize))
72    sram.io.r.req.valid := io.ren(i)
73    sram.io.r.req.bits.setIdx := io.raddr(i)
74    io.rdata(i) := sram.io.r.resp.data(0)
75    sram.io.w.req.valid := io.wen
76    sram.io.w.req.bits.setIdx := io.waddr
77    sram.io.w.req.bits.data := VecInit(io.wdata)
78  }
79
80}
81
82class Ftq_RF_Components(implicit p: Parameters) extends XSBundle with BPUUtils {
83  val startAddr = UInt(VAddrBits.W)
84  val nextLineAddr = UInt(VAddrBits.W)
85  val isNextMask = Vec(PredictWidth, Bool())
86  val fallThruError = Bool()
87  // val carry = Bool()
88  def getPc(offset: UInt) = {
89    def getHigher(pc: UInt) = pc(VAddrBits-1, log2Ceil(PredictWidth)+instOffsetBits+1)
90    def getOffset(pc: UInt) = pc(log2Ceil(PredictWidth)+instOffsetBits, instOffsetBits)
91    Cat(getHigher(Mux(isNextMask(offset) && startAddr(log2Ceil(PredictWidth)+instOffsetBits), nextLineAddr, startAddr)),
92        getOffset(startAddr)+offset, 0.U(instOffsetBits.W))
93  }
94  def fromBranchPrediction(resp: BranchPredictionBundle) = {
95    def carryPos(addr: UInt) = addr(instOffsetBits+log2Ceil(PredictWidth)+1)
96    this.startAddr := resp.pc
97    this.nextLineAddr := resp.pc + (FetchWidth * 4 * 2).U // may be broken on other configs
98    this.isNextMask := VecInit((0 until PredictWidth).map(i =>
99      (resp.pc(log2Ceil(PredictWidth), 1) +& i.U)(log2Ceil(PredictWidth)).asBool()
100    ))
101    this.fallThruError := resp.fallThruError
102    this
103  }
104  override def toPrintable: Printable = {
105    p"startAddr:${Hexadecimal(startAddr)}"
106  }
107}
108
109class Ftq_pd_Entry(implicit p: Parameters) extends XSBundle {
110  val brMask = Vec(PredictWidth, Bool())
111  val jmpInfo = ValidUndirectioned(Vec(3, Bool()))
112  val jmpOffset = UInt(log2Ceil(PredictWidth).W)
113  val jalTarget = UInt(VAddrBits.W)
114  val rvcMask = Vec(PredictWidth, Bool())
115  def hasJal  = jmpInfo.valid && !jmpInfo.bits(0)
116  def hasJalr = jmpInfo.valid && jmpInfo.bits(0)
117  def hasCall = jmpInfo.valid && jmpInfo.bits(1)
118  def hasRet  = jmpInfo.valid && jmpInfo.bits(2)
119
120  def fromPdWb(pdWb: PredecodeWritebackBundle) = {
121    val pds = pdWb.pd
122    this.brMask := VecInit(pds.map(pd => pd.isBr && pd.valid))
123    this.jmpInfo.valid := VecInit(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid)).asUInt.orR
124    this.jmpInfo.bits := ParallelPriorityMux(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid),
125                                             pds.map(pd => VecInit(pd.isJalr, pd.isCall, pd.isRet)))
126    this.jmpOffset := ParallelPriorityEncoder(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid))
127    this.rvcMask := VecInit(pds.map(pd => pd.isRVC))
128    this.jalTarget := pdWb.jalTarget
129  }
130
131  def toPd(offset: UInt) = {
132    require(offset.getWidth == log2Ceil(PredictWidth))
133    val pd = Wire(new PreDecodeInfo)
134    pd.valid := true.B
135    pd.isRVC := rvcMask(offset)
136    val isBr = brMask(offset)
137    val isJalr = offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(0)
138    pd.brType := Cat(offset === jmpOffset && jmpInfo.valid, isJalr || isBr)
139    pd.isCall := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(1)
140    pd.isRet  := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(2)
141    pd
142  }
143}
144
145
146
147class Ftq_Redirect_SRAMEntry(implicit p: Parameters) extends SpeculativeInfo {}
148
149class Ftq_1R_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst {
150  val meta = UInt(MaxMetaLength.W)
151}
152
153class Ftq_Pred_Info(implicit p: Parameters) extends XSBundle {
154  val target = UInt(VAddrBits.W)
155  val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))
156}
157
158
159class FtqRead[T <: Data](private val gen: T)(implicit p: Parameters) extends XSBundle {
160  val ptr = Output(new FtqPtr)
161  val offset = Output(UInt(log2Ceil(PredictWidth).W))
162  val data = Input(gen)
163  def apply(ptr: FtqPtr, offset: UInt) = {
164    this.ptr := ptr
165    this.offset := offset
166    this.data
167  }
168}
169
170
171class FtqToBpuIO(implicit p: Parameters) extends XSBundle {
172  val redirect = Valid(new BranchPredictionRedirect)
173  val update = Valid(new BranchPredictionUpdate)
174  val enq_ptr = Output(new FtqPtr)
175}
176
177class FtqToIfuIO(implicit p: Parameters) extends XSBundle with HasCircularQueuePtrHelper {
178  val req = Decoupled(new FetchRequestBundle)
179  val redirect = Valid(new Redirect)
180  val flushFromBpu = new Bundle {
181    // when ifu pipeline is not stalled,
182    // a packet from bpu s3 can reach f1 at most
183    val s2 = Valid(new FtqPtr)
184    val s3 = Valid(new FtqPtr)
185    def shouldFlushBy(src: Valid[FtqPtr], idx_to_flush: FtqPtr) = {
186      src.valid && !isAfter(src.bits, idx_to_flush)
187    }
188    def shouldFlushByStage2(idx: FtqPtr) = shouldFlushBy(s2, idx)
189    def shouldFlushByStage3(idx: FtqPtr) = shouldFlushBy(s3, idx)
190  }
191}
192
193class FtqToICacheIO(implicit p: Parameters) extends XSBundle with HasCircularQueuePtrHelper {
194  //NOTE: req.bits must be prepare in T cycle
195  // while req.valid is set true in T + 1 cycle
196  val req = Decoupled(new FtqToICacheRequestBundle)
197}
198
199trait HasBackendRedirectInfo extends HasXSParameter {
200  def numRedirectPcRead = exuParameters.JmpCnt + exuParameters.AluCnt + 1
201  def isLoadReplay(r: Valid[Redirect]) = r.bits.flushItself()
202}
203
204class FtqToCtrlIO(implicit p: Parameters) extends XSBundle with HasBackendRedirectInfo {
205  // write to backend pc mem
206  val pc_mem_wen = Output(Bool())
207  val pc_mem_waddr = Output(UInt(log2Ceil(FtqSize).W))
208  val pc_mem_wdata = Output(new Ftq_RF_Components)
209  // newest target
210  val newest_entry_target = Output(UInt(VAddrBits.W))
211  val newest_entry_ptr = Output(new FtqPtr)
212}
213
214
215class FTBEntryGen(implicit p: Parameters) extends XSModule with HasBackendRedirectInfo with HasBPUParameter {
216  val io = IO(new Bundle {
217    val start_addr = Input(UInt(VAddrBits.W))
218    val old_entry = Input(new FTBEntry)
219    val pd = Input(new Ftq_pd_Entry)
220    val cfiIndex = Flipped(Valid(UInt(log2Ceil(PredictWidth).W)))
221    val target = Input(UInt(VAddrBits.W))
222    val hit = Input(Bool())
223    val mispredict_vec = Input(Vec(PredictWidth, Bool()))
224
225    val new_entry = Output(new FTBEntry)
226    val new_br_insert_pos = Output(Vec(numBr, Bool()))
227    val taken_mask = Output(Vec(numBr, Bool()))
228    val jmp_taken = Output(Bool())
229    val mispred_mask = Output(Vec(numBr+1, Bool()))
230
231    // for perf counters
232    val is_init_entry = Output(Bool())
233    val is_old_entry = Output(Bool())
234    val is_new_br = Output(Bool())
235    val is_jalr_target_modified = Output(Bool())
236    val is_always_taken_modified = Output(Bool())
237    val is_br_full = Output(Bool())
238  })
239
240  // no mispredictions detected at predecode
241  val hit = io.hit
242  val pd = io.pd
243
244  val init_entry = WireInit(0.U.asTypeOf(new FTBEntry))
245
246
247  val cfi_is_br = pd.brMask(io.cfiIndex.bits) && io.cfiIndex.valid
248  val entry_has_jmp = pd.jmpInfo.valid
249  val new_jmp_is_jal  = entry_has_jmp && !pd.jmpInfo.bits(0) && io.cfiIndex.valid
250  val new_jmp_is_jalr = entry_has_jmp &&  pd.jmpInfo.bits(0) && io.cfiIndex.valid
251  val new_jmp_is_call = entry_has_jmp &&  pd.jmpInfo.bits(1) && io.cfiIndex.valid
252  val new_jmp_is_ret  = entry_has_jmp &&  pd.jmpInfo.bits(2) && io.cfiIndex.valid
253  val last_jmp_rvi = entry_has_jmp && pd.jmpOffset === (PredictWidth-1).U && !pd.rvcMask.last
254  // val last_br_rvi = cfi_is_br && io.cfiIndex.bits === (PredictWidth-1).U && !pd.rvcMask.last
255
256  val cfi_is_jal = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jal
257  val cfi_is_jalr = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jalr
258
259  def carryPos = log2Ceil(PredictWidth)+instOffsetBits
260  def getLower(pc: UInt) = pc(carryPos-1, instOffsetBits)
261  // if not hit, establish a new entry
262  init_entry.valid := true.B
263  // tag is left for ftb to assign
264
265  // case br
266  val init_br_slot = init_entry.getSlotForBr(0)
267  when (cfi_is_br) {
268    init_br_slot.valid := true.B
269    init_br_slot.offset := io.cfiIndex.bits
270    init_br_slot.setLowerStatByTarget(io.start_addr, io.target, numBr == 1)
271    init_entry.always_taken(0) := true.B // set to always taken on init
272  }
273
274  // case jmp
275  when (entry_has_jmp) {
276    init_entry.tailSlot.offset := pd.jmpOffset
277    init_entry.tailSlot.valid := new_jmp_is_jal || new_jmp_is_jalr
278    init_entry.tailSlot.setLowerStatByTarget(io.start_addr, Mux(cfi_is_jalr, io.target, pd.jalTarget), isShare=false)
279  }
280
281  val jmpPft = getLower(io.start_addr) +& pd.jmpOffset +& Mux(pd.rvcMask(pd.jmpOffset), 1.U, 2.U)
282  init_entry.pftAddr := Mux(entry_has_jmp && !last_jmp_rvi, jmpPft, getLower(io.start_addr))
283  init_entry.carry   := Mux(entry_has_jmp && !last_jmp_rvi, jmpPft(carryPos-instOffsetBits), true.B)
284  init_entry.isJalr := new_jmp_is_jalr
285  init_entry.isCall := new_jmp_is_call
286  init_entry.isRet  := new_jmp_is_ret
287  // that means fall thru points to the middle of an inst
288  init_entry.last_may_be_rvi_call := pd.jmpOffset === (PredictWidth-1).U && !pd.rvcMask(pd.jmpOffset)
289
290  // if hit, check whether a new cfi(only br is possible) is detected
291  val oe = io.old_entry
292  val br_recorded_vec = oe.getBrRecordedVec(io.cfiIndex.bits)
293  val br_recorded = br_recorded_vec.asUInt.orR
294  val is_new_br = cfi_is_br && !br_recorded
295  val new_br_offset = io.cfiIndex.bits
296  // vec(i) means new br will be inserted BEFORE old br(i)
297  val allBrSlotsVec = oe.allSlotsForBr
298  val new_br_insert_onehot = VecInit((0 until numBr).map{
299    i => i match {
300      case 0 =>
301        !allBrSlotsVec(0).valid || new_br_offset < allBrSlotsVec(0).offset
302      case idx =>
303        allBrSlotsVec(idx-1).valid && new_br_offset > allBrSlotsVec(idx-1).offset &&
304        (!allBrSlotsVec(idx).valid || new_br_offset < allBrSlotsVec(idx).offset)
305    }
306  })
307
308  val old_entry_modified = WireInit(io.old_entry)
309  for (i <- 0 until numBr) {
310    val slot = old_entry_modified.allSlotsForBr(i)
311    when (new_br_insert_onehot(i)) {
312      slot.valid := true.B
313      slot.offset := new_br_offset
314      slot.setLowerStatByTarget(io.start_addr, io.target, i == numBr-1)
315      old_entry_modified.always_taken(i) := true.B
316    }.elsewhen (new_br_offset > oe.allSlotsForBr(i).offset) {
317      old_entry_modified.always_taken(i) := false.B
318      // all other fields remain unchanged
319    }.otherwise {
320      // case i == 0, remain unchanged
321      if (i != 0) {
322        val noNeedToMoveFromFormerSlot = (i == numBr-1).B && !oe.brSlots.last.valid
323        when (!noNeedToMoveFromFormerSlot) {
324          slot.fromAnotherSlot(oe.allSlotsForBr(i-1))
325          old_entry_modified.always_taken(i) := oe.always_taken(i)
326        }
327      }
328    }
329  }
330
331  // two circumstances:
332  // 1. oe: | br | j  |, new br should be in front of j, thus addr of j should be new pft
333  // 2. oe: | br | br |, new br could be anywhere between, thus new pft is the addr of either
334  //        the previous last br or the new br
335  val may_have_to_replace = oe.noEmptySlotForNewBr
336  val pft_need_to_change = is_new_br && may_have_to_replace
337  // it should either be the given last br or the new br
338  when (pft_need_to_change) {
339    val new_pft_offset =
340      Mux(!new_br_insert_onehot.asUInt.orR,
341        new_br_offset, oe.allSlotsForBr.last.offset)
342
343    // set jmp to invalid
344    old_entry_modified.pftAddr := getLower(io.start_addr) + new_pft_offset
345    old_entry_modified.carry := (getLower(io.start_addr) +& new_pft_offset).head(1).asBool
346    old_entry_modified.last_may_be_rvi_call := false.B
347    old_entry_modified.isCall := false.B
348    old_entry_modified.isRet := false.B
349    old_entry_modified.isJalr := false.B
350  }
351
352  val old_entry_jmp_target_modified = WireInit(oe)
353  val old_target = oe.tailSlot.getTarget(io.start_addr) // may be wrong because we store only 20 lowest bits
354  val old_tail_is_jmp = !oe.tailSlot.sharing
355  val jalr_target_modified = cfi_is_jalr && (old_target =/= io.target) && old_tail_is_jmp // TODO: pass full jalr target
356  when (jalr_target_modified) {
357    old_entry_jmp_target_modified.setByJmpTarget(io.start_addr, io.target)
358    old_entry_jmp_target_modified.always_taken := 0.U.asTypeOf(Vec(numBr, Bool()))
359  }
360
361  val old_entry_always_taken = WireInit(oe)
362  val always_taken_modified_vec = Wire(Vec(numBr, Bool())) // whether modified or not
363  for (i <- 0 until numBr) {
364    old_entry_always_taken.always_taken(i) :=
365      oe.always_taken(i) && io.cfiIndex.valid && oe.brValids(i) && io.cfiIndex.bits === oe.brOffset(i)
366    always_taken_modified_vec(i) := oe.always_taken(i) && !old_entry_always_taken.always_taken(i)
367  }
368  val always_taken_modified = always_taken_modified_vec.reduce(_||_)
369
370
371
372  val derived_from_old_entry =
373    Mux(is_new_br, old_entry_modified,
374      Mux(jalr_target_modified, old_entry_jmp_target_modified, old_entry_always_taken))
375
376
377  io.new_entry := Mux(!hit, init_entry, derived_from_old_entry)
378
379  io.new_br_insert_pos := new_br_insert_onehot
380  io.taken_mask := VecInit((io.new_entry.brOffset zip io.new_entry.brValids).map{
381    case (off, v) => io.cfiIndex.bits === off && io.cfiIndex.valid && v
382  })
383  io.jmp_taken := io.new_entry.jmpValid && io.new_entry.tailSlot.offset === io.cfiIndex.bits
384  for (i <- 0 until numBr) {
385    io.mispred_mask(i) := io.new_entry.brValids(i) && io.mispredict_vec(io.new_entry.brOffset(i))
386  }
387  io.mispred_mask.last := io.new_entry.jmpValid && io.mispredict_vec(pd.jmpOffset)
388
389  // for perf counters
390  io.is_init_entry := !hit
391  io.is_old_entry := hit && !is_new_br && !jalr_target_modified && !always_taken_modified
392  io.is_new_br := hit && is_new_br
393  io.is_jalr_target_modified := hit && jalr_target_modified
394  io.is_always_taken_modified := hit && always_taken_modified
395  io.is_br_full := hit && is_new_br && may_have_to_replace
396}
397
398class FtqPcMemWrapper(numOtherReads: Int)(implicit p: Parameters) extends XSModule with HasBackendRedirectInfo {
399  val io = IO(new Bundle {
400    val ifuPtr_w       = Input(new FtqPtr)
401    val ifuPtrPlus1_w  = Input(new FtqPtr)
402    val ifuPtrPlus2_w  = Input(new FtqPtr)
403    val commPtr_w      = Input(new FtqPtr)
404    val commPtrPlus1_w = Input(new FtqPtr)
405    val ifuPtr_rdata       = Output(new Ftq_RF_Components)
406    val ifuPtrPlus1_rdata  = Output(new Ftq_RF_Components)
407    val ifuPtrPlus2_rdata  = Output(new Ftq_RF_Components)
408    val commPtr_rdata      = Output(new Ftq_RF_Components)
409    val commPtrPlus1_rdata = Output(new Ftq_RF_Components)
410
411    val other_raddrs = Input(Vec(numOtherReads, UInt(log2Ceil(FtqSize).W)))
412    val other_rdatas = Output(Vec(numOtherReads, new Ftq_RF_Components))
413
414    val wen = Input(Bool())
415    val waddr = Input(UInt(log2Ceil(FtqSize).W))
416    val wdata = Input(new Ftq_RF_Components)
417  })
418
419  val num_pc_read = numOtherReads + 5
420  val mem = Module(new SyncDataModuleTemplate(new Ftq_RF_Components, FtqSize,
421    num_pc_read, 1, "FtqPC"))
422  mem.io.wen(0)   := io.wen
423  mem.io.waddr(0) := io.waddr
424  mem.io.wdata(0) := io.wdata
425
426  // read one cycle ahead for ftq local reads
427  val raddr_vec = VecInit(io.other_raddrs ++
428    Seq(io.ifuPtr_w.value, io.ifuPtrPlus1_w.value, io.ifuPtrPlus2_w.value, io.commPtrPlus1_w.value, io.commPtr_w.value))
429
430  mem.io.raddr := raddr_vec
431
432  io.other_rdatas       := mem.io.rdata.dropRight(5)
433  io.ifuPtr_rdata       := mem.io.rdata.dropRight(4).last
434  io.ifuPtrPlus1_rdata  := mem.io.rdata.dropRight(3).last
435  io.ifuPtrPlus2_rdata  := mem.io.rdata.dropRight(2).last
436  io.commPtrPlus1_rdata := mem.io.rdata.dropRight(1).last
437  io.commPtr_rdata      := mem.io.rdata.last
438}
439
440class Ftq(implicit p: Parameters) extends XSModule with HasCircularQueuePtrHelper
441  with HasBackendRedirectInfo with BPUUtils with HasBPUConst with HasPerfEvents
442  with HasICacheParameters{
443  val io = IO(new Bundle {
444    val fromBpu = Flipped(new BpuToFtqIO)
445    val fromIfu = Flipped(new IfuToFtqIO)
446    val fromBackend = Flipped(new CtrlToFtqIO)
447
448    val toBpu = new FtqToBpuIO
449    val toIfu = new FtqToIfuIO
450    val toICache = new FtqToICacheIO
451    val toBackend = new FtqToCtrlIO
452
453    val toPrefetch = new FtqPrefechBundle
454
455    val bpuInfo = new Bundle {
456      val bpRight = Output(UInt(XLEN.W))
457      val bpWrong = Output(UInt(XLEN.W))
458    }
459
460    val mmioCommitRead = Flipped(new mmioCommitRead)
461  })
462  io.bpuInfo := DontCare
463
464  val backendRedirect = Wire(Valid(new Redirect))
465  val backendRedirectReg = RegNext(backendRedirect)
466
467  val stage2Flush = backendRedirect.valid
468  val backendFlush = stage2Flush || RegNext(stage2Flush)
469  val ifuFlush = Wire(Bool())
470
471  val flush = stage2Flush || RegNext(stage2Flush)
472
473  val allowBpuIn, allowToIfu = WireInit(false.B)
474  val flushToIfu = !allowToIfu
475  allowBpuIn := !ifuFlush && !backendRedirect.valid && !backendRedirectReg.valid
476  allowToIfu := !ifuFlush && !backendRedirect.valid && !backendRedirectReg.valid
477
478  def copyNum = 5
479  val bpuPtr, ifuPtr, ifuWbPtr, commPtr = RegInit(FtqPtr(false.B, 0.U))
480  val ifuPtrPlus1 = RegInit(FtqPtr(false.B, 1.U))
481  val ifuPtrPlus2 = RegInit(FtqPtr(false.B, 2.U))
482  val commPtrPlus1 = RegInit(FtqPtr(false.B, 1.U))
483  val copied_ifu_ptr = Seq.fill(copyNum)(RegInit(FtqPtr(false.B, 0.U)))
484  val copied_bpu_ptr = Seq.fill(copyNum)(RegInit(FtqPtr(false.B, 0.U)))
485  require(FtqSize >= 4)
486  val ifuPtr_write       = WireInit(ifuPtr)
487  val ifuPtrPlus1_write  = WireInit(ifuPtrPlus1)
488  val ifuPtrPlus2_write  = WireInit(ifuPtrPlus2)
489  val ifuWbPtr_write     = WireInit(ifuWbPtr)
490  val commPtr_write      = WireInit(commPtr)
491  val commPtrPlus1_write = WireInit(commPtrPlus1)
492  ifuPtr       := ifuPtr_write
493  ifuPtrPlus1  := ifuPtrPlus1_write
494  ifuPtrPlus2  := ifuPtrPlus2_write
495  ifuWbPtr     := ifuWbPtr_write
496  commPtr      := commPtr_write
497  commPtrPlus1 := commPtrPlus1_write
498  copied_ifu_ptr.map{ptr =>
499    ptr := ifuPtr_write
500    dontTouch(ptr)
501  }
502  val validEntries = distanceBetween(bpuPtr, commPtr)
503
504  // **********************************************************************
505  // **************************** enq from bpu ****************************
506  // **********************************************************************
507  val new_entry_ready = validEntries < FtqSize.U
508  io.fromBpu.resp.ready := new_entry_ready
509
510  val bpu_s2_resp = io.fromBpu.resp.bits.s2
511  val bpu_s3_resp = io.fromBpu.resp.bits.s3
512  val bpu_s2_redirect = bpu_s2_resp.valid && bpu_s2_resp.hasRedirect
513  val bpu_s3_redirect = bpu_s3_resp.valid && bpu_s3_resp.hasRedirect
514
515  io.toBpu.enq_ptr := bpuPtr
516  val enq_fire = io.fromBpu.resp.fire() && allowBpuIn // from bpu s1
517  val bpu_in_fire = (io.fromBpu.resp.fire() || bpu_s2_redirect || bpu_s3_redirect) && allowBpuIn
518
519  val bpu_in_resp = io.fromBpu.resp.bits.selectedResp
520  val bpu_in_stage = io.fromBpu.resp.bits.selectedRespIdx
521  val bpu_in_resp_ptr = Mux(bpu_in_stage === BP_S1, bpuPtr, bpu_in_resp.ftq_idx)
522  val bpu_in_resp_idx = bpu_in_resp_ptr.value
523
524  // read ports:      prefetchReq ++  ifuReq1 + ifuReq2 + ifuReq3 + commitUpdate2 + commitUpdate
525  val ftq_pc_mem = Module(new FtqPcMemWrapper(1))
526  // resp from uBTB
527  ftq_pc_mem.io.wen := bpu_in_fire
528  ftq_pc_mem.io.waddr := bpu_in_resp_idx
529  ftq_pc_mem.io.wdata.fromBranchPrediction(bpu_in_resp)
530
531  //                                                            ifuRedirect + backendRedirect + commit
532  val ftq_redirect_sram = Module(new FtqNRSRAM(new Ftq_Redirect_SRAMEntry, 1+1+1))
533  // these info is intended to enq at the last stage of bpu
534  ftq_redirect_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid
535  ftq_redirect_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value
536  ftq_redirect_sram.io.wdata := io.fromBpu.resp.bits.last_stage_spec_info
537  println(f"ftq redirect SRAM: entry ${ftq_redirect_sram.io.wdata.getWidth} * ${FtqSize} * 3")
538  println(f"ftq redirect SRAM: ahead fh ${ftq_redirect_sram.io.wdata.afhob.getWidth} * ${FtqSize} * 3")
539
540  val ftq_meta_1r_sram = Module(new FtqNRSRAM(new Ftq_1R_SRAMEntry, 1))
541  // these info is intended to enq at the last stage of bpu
542  ftq_meta_1r_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid
543  ftq_meta_1r_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value
544  ftq_meta_1r_sram.io.wdata.meta := io.fromBpu.resp.bits.last_stage_meta
545  //                                                            ifuRedirect + backendRedirect + commit
546  val ftb_entry_mem = Module(new SyncDataModuleTemplate(new FTBEntry, FtqSize, 1+1+1, 1))
547  ftb_entry_mem.io.wen(0) := io.fromBpu.resp.bits.lastStage.valid
548  ftb_entry_mem.io.waddr(0) := io.fromBpu.resp.bits.lastStage.ftq_idx.value
549  ftb_entry_mem.io.wdata(0) := io.fromBpu.resp.bits.last_stage_ftb_entry
550
551
552  // multi-write
553  val update_target = Reg(Vec(FtqSize, UInt(VAddrBits.W))) // could be taken target or fallThrough //TODO: remove this
554  val newest_entry_target = Reg(UInt(VAddrBits.W))
555  val newest_entry_ptr = Reg(new FtqPtr)
556  val cfiIndex_vec = Reg(Vec(FtqSize, ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))))
557  val mispredict_vec = Reg(Vec(FtqSize, Vec(PredictWidth, Bool())))
558  val pred_stage = Reg(Vec(FtqSize, UInt(2.W)))
559
560  val c_invalid :: c_valid :: c_commited :: Nil = Enum(3)
561  val commitStateQueue = RegInit(VecInit(Seq.fill(FtqSize) {
562    VecInit(Seq.fill(PredictWidth)(c_invalid))
563  }))
564
565  val f_to_send :: f_sent :: Nil = Enum(2)
566  val entry_fetch_status = RegInit(VecInit(Seq.fill(FtqSize)(f_sent)))
567
568  val h_not_hit :: h_false_hit :: h_hit :: Nil = Enum(3)
569  val entry_hit_status = RegInit(VecInit(Seq.fill(FtqSize)(h_not_hit)))
570
571  // modify registers one cycle later to cut critical path
572  val last_cycle_bpu_in = RegNext(bpu_in_fire)
573  val last_cycle_bpu_in_ptr = RegNext(bpu_in_resp_ptr)
574  val last_cycle_bpu_in_idx = last_cycle_bpu_in_ptr.value
575  val last_cycle_bpu_target = RegNext(bpu_in_resp.getTarget)
576  val last_cycle_cfiIndex = RegNext(bpu_in_resp.cfiIndex)
577  val last_cycle_bpu_in_stage = RegNext(bpu_in_stage)
578
579  def extra_copyNum_for_commitStateQueue = 2
580  val copied_last_cycle_bpu_in = VecInit(Seq.fill(copyNum+extra_copyNum_for_commitStateQueue)(RegNext(bpu_in_fire)))
581  val copied_last_cycle_bpu_in_ptr_for_ftq = VecInit(Seq.fill(extra_copyNum_for_commitStateQueue)(RegNext(bpu_in_resp_ptr)))
582
583  when (last_cycle_bpu_in) {
584    entry_fetch_status(last_cycle_bpu_in_idx) := f_to_send
585    cfiIndex_vec(last_cycle_bpu_in_idx) := last_cycle_cfiIndex
586    pred_stage(last_cycle_bpu_in_idx) := last_cycle_bpu_in_stage
587
588    update_target(last_cycle_bpu_in_idx) := last_cycle_bpu_target // TODO: remove this
589    newest_entry_target := last_cycle_bpu_target
590    newest_entry_ptr := last_cycle_bpu_in_ptr
591  }
592
593  // reduce fanout by delay write for a cycle
594  when (RegNext(last_cycle_bpu_in)) {
595    mispredict_vec(RegNext(last_cycle_bpu_in_idx)) := WireInit(VecInit(Seq.fill(PredictWidth)(false.B)))
596  }
597
598  // reduce fanout using copied last_cycle_bpu_in and copied last_cycle_bpu_in_ptr
599  val copied_last_cycle_bpu_in_for_ftq = copied_last_cycle_bpu_in.takeRight(extra_copyNum_for_commitStateQueue)
600  copied_last_cycle_bpu_in_for_ftq.zip(copied_last_cycle_bpu_in_ptr_for_ftq).zipWithIndex.map {
601    case ((in, ptr), i) =>
602      when (in) {
603        val perSetEntries = FtqSize / extra_copyNum_for_commitStateQueue // 32
604        require(FtqSize % extra_copyNum_for_commitStateQueue == 0)
605        for (j <- 0 until perSetEntries) {
606          when (ptr.value === (i*perSetEntries+j).U) {
607            commitStateQueue(i*perSetEntries+j) := VecInit(Seq.fill(PredictWidth)(c_invalid))
608          }
609        }
610      }
611  }
612
613  // num cycle is fixed
614  io.toBackend.newest_entry_ptr := RegNext(newest_entry_ptr)
615  io.toBackend.newest_entry_target := RegNext(newest_entry_target)
616
617
618  bpuPtr := bpuPtr + enq_fire
619  copied_bpu_ptr.map(_ := bpuPtr + enq_fire)
620  when (io.toIfu.req.fire && allowToIfu) {
621    ifuPtr_write := ifuPtrPlus1
622    ifuPtrPlus1_write := ifuPtrPlus2
623    ifuPtrPlus2_write := ifuPtrPlus2 + 1.U
624  }
625
626  // only use ftb result to assign hit status
627  when (bpu_s2_resp.valid) {
628    entry_hit_status(bpu_s2_resp.ftq_idx.value) := Mux(bpu_s2_resp.full_pred.hit, h_hit, h_not_hit)
629  }
630
631
632  io.toIfu.flushFromBpu.s2.valid := bpu_s2_redirect
633  io.toIfu.flushFromBpu.s2.bits := bpu_s2_resp.ftq_idx
634  when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect) {
635    bpuPtr := bpu_s2_resp.ftq_idx + 1.U
636    copied_bpu_ptr.map(_ := bpu_s2_resp.ftq_idx + 1.U)
637    // only when ifuPtr runs ahead of bpu s2 resp should we recover it
638    when (!isBefore(ifuPtr, bpu_s2_resp.ftq_idx)) {
639      ifuPtr_write := bpu_s2_resp.ftq_idx
640      ifuPtrPlus1_write := bpu_s2_resp.ftq_idx + 1.U
641      ifuPtrPlus2_write := bpu_s2_resp.ftq_idx + 2.U
642    }
643  }
644
645  io.toIfu.flushFromBpu.s3.valid := bpu_s3_redirect
646  io.toIfu.flushFromBpu.s3.bits := bpu_s3_resp.ftq_idx
647  when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect) {
648    bpuPtr := bpu_s3_resp.ftq_idx + 1.U
649    copied_bpu_ptr.map(_ := bpu_s3_resp.ftq_idx + 1.U)
650    // only when ifuPtr runs ahead of bpu s2 resp should we recover it
651    when (!isBefore(ifuPtr, bpu_s3_resp.ftq_idx)) {
652      ifuPtr_write := bpu_s3_resp.ftq_idx
653      ifuPtrPlus1_write := bpu_s3_resp.ftq_idx + 1.U
654      ifuPtrPlus2_write := bpu_s3_resp.ftq_idx + 2.U
655    }
656  }
657
658  XSError(isBefore(bpuPtr, ifuPtr) && !isFull(bpuPtr, ifuPtr), "\nifuPtr is before bpuPtr!\n")
659
660  (0 until copyNum).map{i =>
661    XSError(copied_bpu_ptr(i) =/= bpuPtr, "\ncopiedBpuPtr is different from bpuPtr!\n")
662  }
663
664  // ****************************************************************
665  // **************************** to ifu ****************************
666  // ****************************************************************
667  // 0  for ifu, and 1-4 for ICache
668  val bpu_in_bypass_buf = RegEnable(ftq_pc_mem.io.wdata, enable=bpu_in_fire)
669  val copied_bpu_in_bypass_buf = VecInit(Seq.fill(copyNum)(RegEnable(ftq_pc_mem.io.wdata, enable=bpu_in_fire)))
670  val bpu_in_bypass_buf_for_ifu = bpu_in_bypass_buf
671  val bpu_in_bypass_ptr = RegNext(bpu_in_resp_ptr)
672  val last_cycle_to_ifu_fire = RegNext(io.toIfu.req.fire)
673
674  val copied_bpu_in_bypass_ptr = VecInit(Seq.fill(copyNum)(RegNext(bpu_in_resp_ptr)))
675  val copied_last_cycle_to_ifu_fire = VecInit(Seq.fill(copyNum)(RegNext(io.toIfu.req.fire)))
676
677  // read pc and target
678  ftq_pc_mem.io.ifuPtr_w       := ifuPtr_write
679  ftq_pc_mem.io.ifuPtrPlus1_w  := ifuPtrPlus1_write
680  ftq_pc_mem.io.ifuPtrPlus2_w  := ifuPtrPlus2_write
681  ftq_pc_mem.io.commPtr_w      := commPtr_write
682  ftq_pc_mem.io.commPtrPlus1_w := commPtrPlus1_write
683
684
685  io.toIfu.req.bits.ftqIdx := ifuPtr
686
687  val toICachePcBundle = Wire(Vec(copyNum,new Ftq_RF_Components))
688  val toICacheEntryToSend = Wire(Vec(copyNum,Bool()))
689  val toIfuPcBundle = Wire(new Ftq_RF_Components)
690  val entry_is_to_send = WireInit(entry_fetch_status(ifuPtr.value) === f_to_send)
691  val entry_ftq_offset = WireInit(cfiIndex_vec(ifuPtr.value))
692  val entry_next_addr  = Wire(UInt(VAddrBits.W))
693
694  val pc_mem_ifu_ptr_rdata   = VecInit(Seq.fill(copyNum)(RegNext(ftq_pc_mem.io.ifuPtr_rdata)))
695  val pc_mem_ifu_plus1_rdata = VecInit(Seq.fill(copyNum)(RegNext(ftq_pc_mem.io.ifuPtrPlus1_rdata)))
696  val diff_entry_next_addr = WireInit(update_target(ifuPtr.value)) //TODO: remove this
697
698  val copied_ifu_plus1_to_send = VecInit(Seq.fill(copyNum)(RegNext(entry_fetch_status(ifuPtrPlus1.value) === f_to_send) || RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1))))
699  val copied_ifu_ptr_to_send   = VecInit(Seq.fill(copyNum)(RegNext(entry_fetch_status(ifuPtr.value) === f_to_send) || RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr)))
700
701  for(i <- 0 until copyNum){
702    when(copied_last_cycle_bpu_in(i) && copied_bpu_in_bypass_ptr(i) === copied_ifu_ptr(i)){
703      toICachePcBundle(i) := copied_bpu_in_bypass_buf(i)
704      toICacheEntryToSend(i)   := true.B
705    }.elsewhen(copied_last_cycle_to_ifu_fire(i)){
706      toICachePcBundle(i) := pc_mem_ifu_plus1_rdata(i)
707      toICacheEntryToSend(i)   := copied_ifu_plus1_to_send(i)
708    }.otherwise{
709      toICachePcBundle(i) := pc_mem_ifu_ptr_rdata(i)
710      toICacheEntryToSend(i)   := copied_ifu_ptr_to_send(i)
711    }
712  }
713
714  // TODO: reconsider target address bypass logic
715  when (last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr) {
716    toIfuPcBundle := bpu_in_bypass_buf_for_ifu
717    entry_is_to_send := true.B
718    entry_next_addr := last_cycle_bpu_target
719    entry_ftq_offset := last_cycle_cfiIndex
720    diff_entry_next_addr := last_cycle_bpu_target // TODO: remove this
721  }.elsewhen (last_cycle_to_ifu_fire) {
722    toIfuPcBundle := RegNext(ftq_pc_mem.io.ifuPtrPlus1_rdata)
723    entry_is_to_send := RegNext(entry_fetch_status(ifuPtrPlus1.value) === f_to_send) ||
724                        RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1)) // reduce potential bubbles
725    entry_next_addr := Mux(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1),
726                          bpu_in_bypass_buf_for_ifu.startAddr,
727                          Mux(ifuPtr === newest_entry_ptr,
728                            newest_entry_target,
729                            RegNext(ftq_pc_mem.io.ifuPtrPlus2_rdata.startAddr))) // ifuPtr+2
730  }.otherwise {
731    toIfuPcBundle := RegNext(ftq_pc_mem.io.ifuPtr_rdata)
732    entry_is_to_send := RegNext(entry_fetch_status(ifuPtr.value) === f_to_send) ||
733                        RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr) // reduce potential bubbles
734    entry_next_addr := Mux(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1),
735                          bpu_in_bypass_buf_for_ifu.startAddr,
736                          Mux(ifuPtr === newest_entry_ptr,
737                            newest_entry_target,
738                            RegNext(ftq_pc_mem.io.ifuPtrPlus1_rdata.startAddr))) // ifuPtr+1
739  }
740
741  io.toIfu.req.valid := entry_is_to_send && ifuPtr =/= bpuPtr
742  io.toIfu.req.bits.nextStartAddr := entry_next_addr
743  io.toIfu.req.bits.ftqOffset := entry_ftq_offset
744  io.toIfu.req.bits.fromFtqPcBundle(toIfuPcBundle)
745
746  io.toICache.req.valid := entry_is_to_send && ifuPtr =/= bpuPtr
747  io.toICache.req.bits.readValid.zipWithIndex.map{case(copy, i) => copy := toICacheEntryToSend(i) && copied_ifu_ptr(i) =/= copied_bpu_ptr(i)}
748  io.toICache.req.bits.pcMemRead.zipWithIndex.map{case(copy,i) => copy.fromFtqPcBundle(toICachePcBundle(i))}
749  // io.toICache.req.bits.bypassSelect := last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr
750  // io.toICache.req.bits.bpuBypassWrite.zipWithIndex.map{case(bypassWrtie, i) =>
751  //   bypassWrtie.startAddr := bpu_in_bypass_buf.tail(i).startAddr
752  //   bypassWrtie.nextlineStart := bpu_in_bypass_buf.tail(i).nextLineAddr
753  // }
754
755  // TODO: remove this
756  XSError(io.toIfu.req.valid && diff_entry_next_addr =/= entry_next_addr,
757          p"\nifu_req_target wrong! ifuPtr: ${ifuPtr}, entry_next_addr: ${Hexadecimal(entry_next_addr)} diff_entry_next_addr: ${Hexadecimal(diff_entry_next_addr)}\n")
758
759  // when fall through is smaller in value than start address, there must be a false hit
760  when (toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit) {
761    when (io.toIfu.req.fire &&
762      !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr) &&
763      !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr)
764    ) {
765      entry_hit_status(ifuPtr.value) := h_false_hit
766      // XSError(true.B, "FTB false hit by fallThroughError, startAddr: %x, fallTHru: %x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr)
767    }
768    XSDebug(true.B, "fallThruError! start:%x, fallThru:%x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr)
769  }
770
771  XSPerfAccumulate(f"fall_through_error_to_ifu", toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit &&
772    io.toIfu.req.fire && !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr) && !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr))
773
774  val ifu_req_should_be_flushed =
775    io.toIfu.flushFromBpu.shouldFlushByStage2(io.toIfu.req.bits.ftqIdx) ||
776    io.toIfu.flushFromBpu.shouldFlushByStage3(io.toIfu.req.bits.ftqIdx)
777
778    when (io.toIfu.req.fire && !ifu_req_should_be_flushed) {
779      entry_fetch_status(ifuPtr.value) := f_sent
780    }
781
782  // *********************************************************************
783  // **************************** wb from ifu ****************************
784  // *********************************************************************
785  val pdWb = io.fromIfu.pdWb
786  val pds = pdWb.bits.pd
787  val ifu_wb_valid = pdWb.valid
788  val ifu_wb_idx = pdWb.bits.ftqIdx.value
789  // read ports:                                                         commit update
790  val ftq_pd_mem = Module(new SyncDataModuleTemplate(new Ftq_pd_Entry, FtqSize, 1, 1))
791  ftq_pd_mem.io.wen(0) := ifu_wb_valid
792  ftq_pd_mem.io.waddr(0) := pdWb.bits.ftqIdx.value
793  ftq_pd_mem.io.wdata(0).fromPdWb(pdWb.bits)
794
795  val hit_pd_valid = entry_hit_status(ifu_wb_idx) === h_hit && ifu_wb_valid
796  val hit_pd_mispred = hit_pd_valid && pdWb.bits.misOffset.valid
797  val hit_pd_mispred_reg = RegNext(hit_pd_mispred, init=false.B)
798  val pd_reg       = RegEnable(pds,             pdWb.valid)
799  val start_pc_reg = RegEnable(pdWb.bits.pc(0), pdWb.valid)
800  val wb_idx_reg   = RegEnable(ifu_wb_idx,      pdWb.valid)
801
802  when (ifu_wb_valid) {
803    val comm_stq_wen = VecInit(pds.map(_.valid).zip(pdWb.bits.instrRange).map{
804      case (v, inRange) => v && inRange
805    })
806    (commitStateQueue(ifu_wb_idx) zip comm_stq_wen).map{
807      case (qe, v) => when (v) { qe := c_valid }
808    }
809  }
810
811  when (ifu_wb_valid) {
812    ifuWbPtr_write := ifuWbPtr + 1.U
813  }
814
815  ftb_entry_mem.io.raddr.head := ifu_wb_idx
816  val has_false_hit = WireInit(false.B)
817  when (RegNext(hit_pd_valid)) {
818    // check for false hit
819    val pred_ftb_entry = ftb_entry_mem.io.rdata.head
820    val brSlots = pred_ftb_entry.brSlots
821    val tailSlot = pred_ftb_entry.tailSlot
822    // we check cfis that bpu predicted
823
824    // bpu predicted branches but denied by predecode
825    val br_false_hit =
826      brSlots.map{
827        s => s.valid && !(pd_reg(s.offset).valid && pd_reg(s.offset).isBr)
828      }.reduce(_||_) ||
829      (tailSlot.valid && pred_ftb_entry.tailSlot.sharing &&
830        !(pd_reg(tailSlot.offset).valid && pd_reg(tailSlot.offset).isBr))
831
832    val jmpOffset = tailSlot.offset
833    val jmp_pd = pd_reg(jmpOffset)
834    val jal_false_hit = pred_ftb_entry.jmpValid &&
835      ((pred_ftb_entry.isJal  && !(jmp_pd.valid && jmp_pd.isJal)) ||
836       (pred_ftb_entry.isJalr && !(jmp_pd.valid && jmp_pd.isJalr)) ||
837       (pred_ftb_entry.isCall && !(jmp_pd.valid && jmp_pd.isCall)) ||
838       (pred_ftb_entry.isRet  && !(jmp_pd.valid && jmp_pd.isRet))
839      )
840
841    has_false_hit := br_false_hit || jal_false_hit || hit_pd_mispred_reg
842    XSDebug(has_false_hit, "FTB false hit by br or jal or hit_pd, startAddr: %x\n", pdWb.bits.pc(0))
843
844    // assert(!has_false_hit)
845  }
846
847  when (has_false_hit) {
848    entry_hit_status(wb_idx_reg) := h_false_hit
849  }
850
851
852  // **********************************************************************
853  // ***************************** to backend *****************************
854  // **********************************************************************
855  // to backend pc mem / target
856  io.toBackend.pc_mem_wen   := RegNext(last_cycle_bpu_in)
857  io.toBackend.pc_mem_waddr := RegNext(last_cycle_bpu_in_idx)
858  io.toBackend.pc_mem_wdata := RegNext(bpu_in_bypass_buf_for_ifu)
859
860  // *******************************************************************************
861  // **************************** redirect from backend ****************************
862  // *******************************************************************************
863
864  // redirect read cfiInfo, couples to redirectGen s2
865  ftq_redirect_sram.io.ren.init.last := backendRedirect.valid
866  ftq_redirect_sram.io.raddr.init.last := backendRedirect.bits.ftqIdx.value
867
868  ftb_entry_mem.io.raddr.init.last := backendRedirect.bits.ftqIdx.value
869
870  val stage3CfiInfo = ftq_redirect_sram.io.rdata.init.last
871  val fromBackendRedirect = WireInit(backendRedirectReg)
872  val backendRedirectCfi = fromBackendRedirect.bits.cfiUpdate
873  backendRedirectCfi.fromFtqRedirectSram(stage3CfiInfo)
874
875  val r_ftb_entry = ftb_entry_mem.io.rdata.init.last
876  val r_ftqOffset = fromBackendRedirect.bits.ftqOffset
877
878  when (entry_hit_status(fromBackendRedirect.bits.ftqIdx.value) === h_hit) {
879    backendRedirectCfi.shift := PopCount(r_ftb_entry.getBrMaskByOffset(r_ftqOffset)) +&
880      (backendRedirectCfi.pd.isBr && !r_ftb_entry.brIsSaved(r_ftqOffset) &&
881      !r_ftb_entry.newBrCanNotInsert(r_ftqOffset))
882
883    backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr && (r_ftb_entry.brIsSaved(r_ftqOffset) ||
884        !r_ftb_entry.newBrCanNotInsert(r_ftqOffset))
885  }.otherwise {
886    backendRedirectCfi.shift := (backendRedirectCfi.pd.isBr && backendRedirectCfi.taken).asUInt
887    backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr.asUInt
888  }
889
890
891  // ***************************************************************************
892  // **************************** redirect from ifu ****************************
893  // ***************************************************************************
894  val fromIfuRedirect = WireInit(0.U.asTypeOf(Valid(new Redirect)))
895  fromIfuRedirect.valid := pdWb.valid && pdWb.bits.misOffset.valid && !backendFlush
896  fromIfuRedirect.bits.ftqIdx := pdWb.bits.ftqIdx
897  fromIfuRedirect.bits.ftqOffset := pdWb.bits.misOffset.bits
898  fromIfuRedirect.bits.level := RedirectLevel.flushAfter
899
900  val ifuRedirectCfiUpdate = fromIfuRedirect.bits.cfiUpdate
901  ifuRedirectCfiUpdate.pc := pdWb.bits.pc(pdWb.bits.misOffset.bits)
902  ifuRedirectCfiUpdate.pd := pdWb.bits.pd(pdWb.bits.misOffset.bits)
903  ifuRedirectCfiUpdate.predTaken := cfiIndex_vec(pdWb.bits.ftqIdx.value).valid
904  ifuRedirectCfiUpdate.target := pdWb.bits.target
905  ifuRedirectCfiUpdate.taken := pdWb.bits.cfiOffset.valid
906  ifuRedirectCfiUpdate.isMisPred := pdWb.bits.misOffset.valid
907
908  val ifuRedirectReg = RegNext(fromIfuRedirect, init=0.U.asTypeOf(Valid(new Redirect)))
909  val ifuRedirectToBpu = WireInit(ifuRedirectReg)
910  ifuFlush := fromIfuRedirect.valid || ifuRedirectToBpu.valid
911
912  ftq_redirect_sram.io.ren.head := fromIfuRedirect.valid
913  ftq_redirect_sram.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value
914
915  ftb_entry_mem.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value
916
917  val toBpuCfi = ifuRedirectToBpu.bits.cfiUpdate
918  toBpuCfi.fromFtqRedirectSram(ftq_redirect_sram.io.rdata.head)
919  when (ifuRedirectReg.bits.cfiUpdate.pd.isRet) {
920    toBpuCfi.target := toBpuCfi.rasEntry.retAddr
921  }
922
923  // *********************************************************************
924  // **************************** wb from exu ****************************
925  // *********************************************************************
926
927  backendRedirect := io.fromBackend.redirect
928
929  def extractRedirectInfo(wb: Valid[Redirect]) = {
930    val ftqPtr = wb.bits.ftqIdx
931    val ftqOffset = wb.bits.ftqOffset
932    val taken = wb.bits.cfiUpdate.taken
933    val mispred = wb.bits.cfiUpdate.isMisPred
934    (wb.valid, ftqPtr, ftqOffset, taken, mispred)
935  }
936
937  // fix mispredict entry
938  val lastIsMispredict = RegNext(
939    backendRedirect.valid && backendRedirect.bits.level === RedirectLevel.flushAfter, init = false.B
940  )
941
942  def updateCfiInfo(redirect: Valid[Redirect], isBackend: Boolean = true) = {
943    val (r_valid, r_ptr, r_offset, r_taken, r_mispred) = extractRedirectInfo(redirect)
944    val r_idx = r_ptr.value
945    val cfiIndex_bits_wen = r_valid && r_taken && r_offset < cfiIndex_vec(r_idx).bits
946    val cfiIndex_valid_wen = r_valid && r_offset === cfiIndex_vec(r_idx).bits
947    when (cfiIndex_bits_wen || cfiIndex_valid_wen) {
948      cfiIndex_vec(r_idx).valid := cfiIndex_bits_wen || cfiIndex_valid_wen && r_taken
949    } .elsewhen (r_valid && !r_taken && r_offset =/= cfiIndex_vec(r_idx).bits) {
950      cfiIndex_vec(r_idx).valid :=false.B
951    }
952    when (cfiIndex_bits_wen) {
953      cfiIndex_vec(r_idx).bits := r_offset
954    }
955    newest_entry_target := redirect.bits.cfiUpdate.target
956    newest_entry_ptr := r_ptr
957    update_target(r_idx) := redirect.bits.cfiUpdate.target // TODO: remove this
958    if (isBackend) {
959      mispredict_vec(r_idx)(r_offset) := r_mispred
960    }
961  }
962
963  when(backendRedirectReg.valid) {
964    updateCfiInfo(backendRedirectReg)
965  }.elsewhen (ifuRedirectToBpu.valid) {
966    updateCfiInfo(ifuRedirectToBpu, isBackend=false)
967  }
968
969  // ***********************************************************************************
970  // **************************** flush ptr and state queue ****************************
971  // ***********************************************************************************
972
973  val redirectVec = VecInit(backendRedirect, fromIfuRedirect)
974
975  // when redirect, we should reset ptrs and status queues
976  when(redirectVec.map(r => r.valid).reduce(_||_)){
977    val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits)))
978    val notIfu = redirectVec.dropRight(1).map(r => r.valid).reduce(_||_)
979    val (idx, offset, flushItSelf) = (r.ftqIdx, r.ftqOffset, RedirectLevel.flushItself(r.level))
980    val next = idx + 1.U
981    bpuPtr := next
982    copied_bpu_ptr.map(_ := next)
983    ifuPtr_write := next
984    ifuWbPtr_write := next
985    ifuPtrPlus1_write := idx + 2.U
986    ifuPtrPlus2_write := idx + 3.U
987
988  }
989  when(RegNext(redirectVec.map(r => r.valid).reduce(_||_))){
990    val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits)))
991    val notIfu = redirectVec.dropRight(1).map(r => r.valid).reduce(_||_)
992    val (idx, offset, flushItSelf) = (r.ftqIdx, r.ftqOffset, RedirectLevel.flushItself(r.level))
993    when (RegNext(notIfu)) {
994      commitStateQueue(RegNext(idx.value)).zipWithIndex.foreach({ case (s, i) =>
995        when(i.U > RegNext(offset) || i.U === RegNext(offset) && RegNext(flushItSelf)){
996          s := c_invalid
997        }
998      })
999    }
1000  }
1001
1002
1003  // only the valid bit is actually needed
1004  io.toIfu.redirect.bits    := backendRedirect.bits
1005  io.toIfu.redirect.valid   := stage2Flush
1006
1007  // commit
1008  for (c <- io.fromBackend.rob_commits) {
1009    when(c.valid) {
1010      commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset) := c_commited
1011      // TODO: remove this
1012      // For instruction fusions, we also update the next instruction
1013      when (c.bits.commitType === 4.U) {
1014        commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 1.U) := c_commited
1015      }.elsewhen(c.bits.commitType === 5.U) {
1016        commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 2.U) := c_commited
1017      }.elsewhen(c.bits.commitType === 6.U) {
1018        val index = (c.bits.ftqIdx + 1.U).value
1019        commitStateQueue(index)(0) := c_commited
1020      }.elsewhen(c.bits.commitType === 7.U) {
1021        val index = (c.bits.ftqIdx + 1.U).value
1022        commitStateQueue(index)(1) := c_commited
1023      }
1024    }
1025  }
1026
1027  // ****************************************************************
1028  // **************************** to bpu ****************************
1029  // ****************************************************************
1030
1031  io.toBpu.redirect <> Mux(fromBackendRedirect.valid, fromBackendRedirect, ifuRedirectToBpu)
1032
1033  val may_have_stall_from_bpu = Wire(Bool())
1034  val bpu_ftb_update_stall = RegInit(0.U(2.W)) // 2-cycle stall, so we need 3 states
1035  may_have_stall_from_bpu := bpu_ftb_update_stall =/= 0.U
1036  val canCommit = commPtr =/= ifuWbPtr && !may_have_stall_from_bpu &&
1037    Cat(commitStateQueue(commPtr.value).map(s => {
1038      s === c_invalid || s === c_commited
1039    })).andR()
1040
1041  val mmioReadPtr = io.mmioCommitRead.mmioFtqPtr
1042  val mmioLastCommit = isBefore(commPtr, mmioReadPtr) && (isAfter(ifuPtr,mmioReadPtr)  ||  mmioReadPtr ===   ifuPtr) &&
1043                       Cat(commitStateQueue(mmioReadPtr.value).map(s => { s === c_invalid || s === c_commited})).andR()
1044  io.mmioCommitRead.mmioLastCommit := RegNext(mmioLastCommit)
1045
1046  // commit reads
1047  val commit_pc_bundle = RegNext(ftq_pc_mem.io.commPtr_rdata)
1048  val commit_target =
1049    Mux(RegNext(commPtr === newest_entry_ptr),
1050      RegNext(newest_entry_target),
1051      RegNext(ftq_pc_mem.io.commPtrPlus1_rdata.startAddr))
1052  ftq_pd_mem.io.raddr.last := commPtr.value
1053  val commit_pd = ftq_pd_mem.io.rdata.last
1054  ftq_redirect_sram.io.ren.last := canCommit
1055  ftq_redirect_sram.io.raddr.last := commPtr.value
1056  val commit_spec_meta = ftq_redirect_sram.io.rdata.last
1057  ftq_meta_1r_sram.io.ren(0) := canCommit
1058  ftq_meta_1r_sram.io.raddr(0) := commPtr.value
1059  val commit_meta = ftq_meta_1r_sram.io.rdata(0)
1060  ftb_entry_mem.io.raddr.last := commPtr.value
1061  val commit_ftb_entry = ftb_entry_mem.io.rdata.last
1062
1063  // need one cycle to read mem and srams
1064  val do_commit_ptr = RegNext(commPtr)
1065  val do_commit = RegNext(canCommit, init=false.B)
1066  when (canCommit) {
1067    commPtr_write := commPtrPlus1
1068    commPtrPlus1_write := commPtrPlus1 + 1.U
1069  }
1070  val commit_state = RegNext(commitStateQueue(commPtr.value))
1071  val can_commit_cfi = WireInit(cfiIndex_vec(commPtr.value))
1072  //
1073  //when (commitStateQueue(commPtr.value)(can_commit_cfi.bits) =/= c_commited) {
1074  //  can_commit_cfi.valid := false.B
1075  //}
1076  val commit_cfi = RegNext(can_commit_cfi)
1077  val debug_cfi = RegNext(commitStateQueue(commPtr.value)(can_commit_cfi.bits) =/= c_commited && can_commit_cfi.valid)
1078
1079  val commit_mispredict = VecInit((RegNext(mispredict_vec(commPtr.value)) zip commit_state).map {
1080    case (mis, state) => mis && state === c_commited
1081  })
1082  val can_commit_hit = entry_hit_status(commPtr.value)
1083  val commit_hit = RegNext(can_commit_hit)
1084  val diff_commit_target = RegNext(update_target(commPtr.value)) // TODO: remove this
1085  val commit_stage = RegNext(pred_stage(commPtr.value))
1086  val commit_valid = commit_hit === h_hit || commit_cfi.valid // hit or taken
1087
1088  val to_bpu_hit = can_commit_hit === h_hit || can_commit_hit === h_false_hit
1089  switch (bpu_ftb_update_stall) {
1090    is (0.U) {
1091      when (can_commit_cfi.valid && !to_bpu_hit && canCommit) {
1092        bpu_ftb_update_stall := 2.U // 2-cycle stall
1093      }
1094    }
1095    is (2.U) {
1096      bpu_ftb_update_stall := 1.U
1097    }
1098    is (1.U) {
1099      bpu_ftb_update_stall := 0.U
1100    }
1101    is (3.U) {
1102      XSError(true.B, "bpu_ftb_update_stall should be 0, 1 or 2")
1103    }
1104  }
1105
1106  // TODO: remove this
1107  XSError(do_commit && diff_commit_target =/= commit_target, "\ncommit target should be the same as update target\n")
1108
1109  io.toBpu.update := DontCare
1110  io.toBpu.update.valid := commit_valid && do_commit
1111  val update = io.toBpu.update.bits
1112  update.false_hit   := commit_hit === h_false_hit
1113  update.pc          := commit_pc_bundle.startAddr
1114  update.meta        := commit_meta.meta
1115  update.cfi_idx     := commit_cfi
1116  update.full_target := commit_target
1117  update.from_stage  := commit_stage
1118  update.spec_info   := commit_spec_meta
1119  XSError(commit_valid && do_commit && debug_cfi, "\ncommit cfi can be non c_commited\n")
1120
1121  val commit_real_hit = commit_hit === h_hit
1122  val update_ftb_entry = update.ftb_entry
1123
1124  val ftbEntryGen = Module(new FTBEntryGen).io
1125  ftbEntryGen.start_addr     := commit_pc_bundle.startAddr
1126  ftbEntryGen.old_entry      := commit_ftb_entry
1127  ftbEntryGen.pd             := commit_pd
1128  ftbEntryGen.cfiIndex       := commit_cfi
1129  ftbEntryGen.target         := commit_target
1130  ftbEntryGen.hit            := commit_real_hit
1131  ftbEntryGen.mispredict_vec := commit_mispredict
1132
1133  update_ftb_entry         := ftbEntryGen.new_entry
1134  update.new_br_insert_pos := ftbEntryGen.new_br_insert_pos
1135  update.mispred_mask      := ftbEntryGen.mispred_mask
1136  update.old_entry         := ftbEntryGen.is_old_entry
1137  update.pred_hit          := commit_hit === h_hit || commit_hit === h_false_hit
1138  update.br_taken_mask     := ftbEntryGen.taken_mask
1139  update.jmp_taken         := ftbEntryGen.jmp_taken
1140
1141  // update.full_pred.fromFtbEntry(ftbEntryGen.new_entry, update.pc)
1142  // update.full_pred.jalr_target := commit_target
1143  // update.full_pred.hit := true.B
1144  // when (update.full_pred.is_jalr) {
1145  //   update.full_pred.targets.last := commit_target
1146  // }
1147
1148  // ****************************************************************
1149  // *********************** to prefetch ****************************
1150  // ****************************************************************
1151
1152  ftq_pc_mem.io.other_raddrs(0) := DontCare
1153  if(cacheParams.hasPrefetch){
1154    val prefetchPtr = RegInit(FtqPtr(false.B, 0.U))
1155    val diff_prefetch_addr = WireInit(update_target(prefetchPtr.value)) //TODO: remove this
1156
1157    prefetchPtr := prefetchPtr + io.toPrefetch.req.fire()
1158
1159    ftq_pc_mem.io.other_raddrs(0) := prefetchPtr.value
1160
1161    when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect && !isBefore(prefetchPtr, bpu_s2_resp.ftq_idx)) {
1162      prefetchPtr := bpu_s2_resp.ftq_idx
1163    }
1164
1165    when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect && !isBefore(prefetchPtr, bpu_s3_resp.ftq_idx)) {
1166      prefetchPtr := bpu_s3_resp.ftq_idx
1167      // XSError(true.B, "\ns3_redirect mechanism not implemented!\n")
1168    }
1169
1170
1171    val prefetch_is_to_send = WireInit(entry_fetch_status(prefetchPtr.value) === f_to_send)
1172    val prefetch_addr = Wire(UInt(VAddrBits.W))
1173
1174    when (last_cycle_bpu_in && bpu_in_bypass_ptr === prefetchPtr) {
1175      prefetch_is_to_send := true.B
1176      prefetch_addr := last_cycle_bpu_target
1177      diff_prefetch_addr := last_cycle_bpu_target // TODO: remove this
1178    }.otherwise{
1179      prefetch_addr := RegNext( ftq_pc_mem.io.other_rdatas(0).startAddr)
1180    }
1181    io.toPrefetch.req.valid := prefetchPtr =/= bpuPtr && prefetch_is_to_send
1182    io.toPrefetch.req.bits.target := prefetch_addr
1183
1184    when(redirectVec.map(r => r.valid).reduce(_||_)){
1185      val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits)))
1186      val next = r.ftqIdx + 1.U
1187      prefetchPtr := next
1188    }
1189
1190    // TODO: remove this
1191    // XSError(io.toPrefetch.req.valid && diff_prefetch_addr =/= prefetch_addr,
1192    //         f"\nprefetch_req_target wrong! prefetchPtr: ${prefetchPtr}, prefetch_addr: ${Hexadecimal(prefetch_addr)} diff_prefetch_addr: ${Hexadecimal(diff_prefetch_addr)}\n")
1193
1194
1195    XSError(isBefore(bpuPtr, prefetchPtr) && !isFull(bpuPtr, prefetchPtr), "\nprefetchPtr is before bpuPtr!\n")
1196    XSError(isBefore(prefetchPtr, ifuPtr) && !isFull(ifuPtr, prefetchPtr), "\nifuPtr is before prefetchPtr!\n")
1197  }
1198  else {
1199    io.toPrefetch.req <> DontCare
1200  }
1201
1202  // ******************************************************************************
1203  // **************************** commit perf counters ****************************
1204  // ******************************************************************************
1205
1206  val commit_inst_mask    = VecInit(commit_state.map(c => c === c_commited && do_commit)).asUInt
1207  val commit_mispred_mask = commit_mispredict.asUInt
1208  val commit_not_mispred_mask = ~commit_mispred_mask
1209
1210  val commit_br_mask = commit_pd.brMask.asUInt
1211  val commit_jmp_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.jmpInfo.valid.asTypeOf(UInt(1.W)))
1212  val commit_cfi_mask = (commit_br_mask | commit_jmp_mask)
1213
1214  val mbpInstrs = commit_inst_mask & commit_cfi_mask
1215
1216  val mbpRights = mbpInstrs & commit_not_mispred_mask
1217  val mbpWrongs = mbpInstrs & commit_mispred_mask
1218
1219  io.bpuInfo.bpRight := PopCount(mbpRights)
1220  io.bpuInfo.bpWrong := PopCount(mbpWrongs)
1221
1222  val ftqBranchTraceDB = ChiselDB.createTable("FTQTable" + p(XSCoreParamsKey).HartId.toString, new FtqDebugBundle)
1223  // Cfi Info
1224  for (i <- 0 until PredictWidth) {
1225    val pc = commit_pc_bundle.startAddr + (i * instBytes).U
1226    val v = commit_state(i) === c_commited
1227    val isBr = commit_pd.brMask(i)
1228    val isJmp = commit_pd.jmpInfo.valid && commit_pd.jmpOffset === i.U
1229    val isCfi = isBr || isJmp
1230    val isTaken = commit_cfi.valid && commit_cfi.bits === i.U
1231    val misPred = commit_mispredict(i)
1232    // val ghist = commit_spec_meta.ghist.predHist
1233    val histPtr = commit_spec_meta.histPtr
1234    val predCycle = commit_meta.meta(63, 0)
1235    val target = commit_target
1236
1237    val brIdx = OHToUInt(Reverse(Cat(update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U})))
1238    val inFtbEntry = update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U}.reduce(_||_)
1239    val addIntoHist = ((commit_hit === h_hit) && inFtbEntry) || ((!(commit_hit === h_hit) && i.U === commit_cfi.bits && isBr && commit_cfi.valid))
1240    XSDebug(v && do_commit && isCfi, p"cfi_update: isBr(${isBr}) pc(${Hexadecimal(pc)}) " +
1241    p"taken(${isTaken}) mispred(${misPred}) cycle($predCycle) hist(${histPtr.value}) " +
1242    p"startAddr(${Hexadecimal(commit_pc_bundle.startAddr)}) AddIntoHist(${addIntoHist}) " +
1243    p"brInEntry(${inFtbEntry}) brIdx(${brIdx}) target(${Hexadecimal(target)})\n")
1244
1245    val logbundle = Wire(new FtqDebugBundle)
1246    logbundle.pc := pc
1247    logbundle.target := target
1248    logbundle.isBr := isBr
1249    logbundle.isJmp := isJmp
1250    logbundle.isCall := isJmp && commit_pd.hasCall
1251    logbundle.isRet := isJmp && commit_pd.hasRet
1252    logbundle.misPred := misPred
1253    logbundle.isTaken := isTaken
1254    logbundle.predStage := commit_stage
1255
1256    ftqBranchTraceDB.log(
1257      data = logbundle /* hardware of type T */,
1258      en = v && do_commit && isCfi,
1259      site = "FTQ" + p(XSCoreParamsKey).HartId.toString,
1260      clock = clock,
1261      reset = reset
1262    )
1263  }
1264
1265  val enq = io.fromBpu.resp
1266  val perf_redirect = backendRedirect
1267
1268  XSPerfAccumulate("entry", validEntries)
1269  XSPerfAccumulate("bpu_to_ftq_stall", enq.valid && !enq.ready)
1270  XSPerfAccumulate("mispredictRedirect", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level)
1271  XSPerfAccumulate("replayRedirect", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level))
1272  XSPerfAccumulate("predecodeRedirect", fromIfuRedirect.valid)
1273
1274  XSPerfAccumulate("to_ifu_bubble", io.toIfu.req.ready && !io.toIfu.req.valid)
1275
1276  XSPerfAccumulate("to_ifu_stall", io.toIfu.req.valid && !io.toIfu.req.ready)
1277  XSPerfAccumulate("from_bpu_real_bubble", !enq.valid && enq.ready && allowBpuIn)
1278  XSPerfAccumulate("bpu_to_ifu_bubble", bpuPtr === ifuPtr)
1279
1280  val from_bpu = io.fromBpu.resp.bits
1281  def in_entry_len_map_gen(resp: BpuToFtqBundle)(stage: String) = {
1282    val entry_len = (resp.last_stage_ftb_entry.getFallThrough(resp.s3.pc) - resp.s3.pc) >> instOffsetBits
1283    val entry_len_recording_vec = (1 to PredictWidth+1).map(i => entry_len === i.U)
1284    val entry_len_map = (1 to PredictWidth+1).map(i =>
1285      f"${stage}_ftb_entry_len_$i" -> (entry_len_recording_vec(i-1) && resp.s3.valid)
1286    ).foldLeft(Map[String, UInt]())(_+_)
1287    entry_len_map
1288  }
1289  val s3_entry_len_map = in_entry_len_map_gen(from_bpu)("s3")
1290
1291  val to_ifu = io.toIfu.req.bits
1292
1293
1294
1295  val commit_num_inst_recording_vec = (1 to PredictWidth).map(i => PopCount(commit_inst_mask) === i.U)
1296  val commit_num_inst_map = (1 to PredictWidth).map(i =>
1297    f"commit_num_inst_$i" -> (commit_num_inst_recording_vec(i-1) && do_commit)
1298  ).foldLeft(Map[String, UInt]())(_+_)
1299
1300
1301
1302  val commit_jal_mask  = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJal.asTypeOf(UInt(1.W)))
1303  val commit_jalr_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJalr.asTypeOf(UInt(1.W)))
1304  val commit_call_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasCall.asTypeOf(UInt(1.W)))
1305  val commit_ret_mask  = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasRet.asTypeOf(UInt(1.W)))
1306
1307
1308  val mbpBRights = mbpRights & commit_br_mask
1309  val mbpJRights = mbpRights & commit_jal_mask
1310  val mbpIRights = mbpRights & commit_jalr_mask
1311  val mbpCRights = mbpRights & commit_call_mask
1312  val mbpRRights = mbpRights & commit_ret_mask
1313
1314  val mbpBWrongs = mbpWrongs & commit_br_mask
1315  val mbpJWrongs = mbpWrongs & commit_jal_mask
1316  val mbpIWrongs = mbpWrongs & commit_jalr_mask
1317  val mbpCWrongs = mbpWrongs & commit_call_mask
1318  val mbpRWrongs = mbpWrongs & commit_ret_mask
1319
1320  val commit_pred_stage = RegNext(pred_stage(commPtr.value))
1321
1322  def pred_stage_map(src: UInt, name: String) = {
1323    (0 until numBpStages).map(i =>
1324      f"${name}_stage_${i+1}" -> PopCount(src.asBools.map(_ && commit_pred_stage === BP_STAGES(i)))
1325    ).foldLeft(Map[String, UInt]())(_+_)
1326  }
1327
1328  val mispred_stage_map      = pred_stage_map(mbpWrongs,  "mispredict")
1329  val br_mispred_stage_map   = pred_stage_map(mbpBWrongs, "br_mispredict")
1330  val jalr_mispred_stage_map = pred_stage_map(mbpIWrongs, "jalr_mispredict")
1331  val correct_stage_map      = pred_stage_map(mbpRights,  "correct")
1332  val br_correct_stage_map   = pred_stage_map(mbpBRights, "br_correct")
1333  val jalr_correct_stage_map = pred_stage_map(mbpIRights, "jalr_correct")
1334
1335  val update_valid = io.toBpu.update.valid
1336  def u(cond: Bool) = update_valid && cond
1337  val ftb_false_hit = u(update.false_hit)
1338  // assert(!ftb_false_hit)
1339  val ftb_hit = u(commit_hit === h_hit)
1340
1341  val ftb_new_entry = u(ftbEntryGen.is_init_entry)
1342  val ftb_new_entry_only_br = ftb_new_entry && !update_ftb_entry.jmpValid
1343  val ftb_new_entry_only_jmp = ftb_new_entry && !update_ftb_entry.brValids(0)
1344  val ftb_new_entry_has_br_and_jmp = ftb_new_entry && update_ftb_entry.brValids(0) && update_ftb_entry.jmpValid
1345
1346  val ftb_old_entry = u(ftbEntryGen.is_old_entry)
1347
1348  val ftb_modified_entry = u(ftbEntryGen.is_new_br || ftbEntryGen.is_jalr_target_modified || ftbEntryGen.is_always_taken_modified)
1349  val ftb_modified_entry_new_br = u(ftbEntryGen.is_new_br)
1350  val ftb_modified_entry_jalr_target_modified = u(ftbEntryGen.is_jalr_target_modified)
1351  val ftb_modified_entry_br_full = ftb_modified_entry && ftbEntryGen.is_br_full
1352  val ftb_modified_entry_always_taken = ftb_modified_entry && ftbEntryGen.is_always_taken_modified
1353
1354  val ftb_entry_len = (ftbEntryGen.new_entry.getFallThrough(update.pc) - update.pc) >> instOffsetBits
1355  val ftb_entry_len_recording_vec = (1 to PredictWidth+1).map(i => ftb_entry_len === i.U)
1356  val ftb_init_entry_len_map = (1 to PredictWidth+1).map(i =>
1357    f"ftb_init_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_new_entry)
1358  ).foldLeft(Map[String, UInt]())(_+_)
1359  val ftb_modified_entry_len_map = (1 to PredictWidth+1).map(i =>
1360    f"ftb_modified_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_modified_entry)
1361  ).foldLeft(Map[String, UInt]())(_+_)
1362
1363  val ftq_occupancy_map = (0 to FtqSize).map(i =>
1364    f"ftq_has_entry_$i" ->( validEntries === i.U)
1365  ).foldLeft(Map[String, UInt]())(_+_)
1366
1367  val perfCountsMap = Map(
1368    "BpInstr" -> PopCount(mbpInstrs),
1369    "BpBInstr" -> PopCount(mbpBRights | mbpBWrongs),
1370    "BpRight"  -> PopCount(mbpRights),
1371    "BpWrong"  -> PopCount(mbpWrongs),
1372    "BpBRight" -> PopCount(mbpBRights),
1373    "BpBWrong" -> PopCount(mbpBWrongs),
1374    "BpJRight" -> PopCount(mbpJRights),
1375    "BpJWrong" -> PopCount(mbpJWrongs),
1376    "BpIRight" -> PopCount(mbpIRights),
1377    "BpIWrong" -> PopCount(mbpIWrongs),
1378    "BpCRight" -> PopCount(mbpCRights),
1379    "BpCWrong" -> PopCount(mbpCWrongs),
1380    "BpRRight" -> PopCount(mbpRRights),
1381    "BpRWrong" -> PopCount(mbpRWrongs),
1382
1383    "ftb_false_hit"                -> PopCount(ftb_false_hit),
1384    "ftb_hit"                      -> PopCount(ftb_hit),
1385    "ftb_new_entry"                -> PopCount(ftb_new_entry),
1386    "ftb_new_entry_only_br"        -> PopCount(ftb_new_entry_only_br),
1387    "ftb_new_entry_only_jmp"       -> PopCount(ftb_new_entry_only_jmp),
1388    "ftb_new_entry_has_br_and_jmp" -> PopCount(ftb_new_entry_has_br_and_jmp),
1389    "ftb_old_entry"                -> PopCount(ftb_old_entry),
1390    "ftb_modified_entry"           -> PopCount(ftb_modified_entry),
1391    "ftb_modified_entry_new_br"    -> PopCount(ftb_modified_entry_new_br),
1392    "ftb_jalr_target_modified"     -> PopCount(ftb_modified_entry_jalr_target_modified),
1393    "ftb_modified_entry_br_full"   -> PopCount(ftb_modified_entry_br_full),
1394    "ftb_modified_entry_always_taken" -> PopCount(ftb_modified_entry_always_taken)
1395  ) ++ ftb_init_entry_len_map ++ ftb_modified_entry_len_map ++
1396  s3_entry_len_map ++ commit_num_inst_map ++ ftq_occupancy_map ++
1397  mispred_stage_map ++ br_mispred_stage_map ++ jalr_mispred_stage_map ++
1398  correct_stage_map ++ br_correct_stage_map ++ jalr_correct_stage_map
1399
1400  for((key, value) <- perfCountsMap) {
1401    XSPerfAccumulate(key, value)
1402  }
1403
1404  // --------------------------- Debug --------------------------------
1405  // XSDebug(enq_fire, p"enq! " + io.fromBpu.resp.bits.toPrintable)
1406  XSDebug(io.toIfu.req.fire, p"fire to ifu " + io.toIfu.req.bits.toPrintable)
1407  XSDebug(do_commit, p"deq! [ptr] $do_commit_ptr\n")
1408  XSDebug(true.B, p"[bpuPtr] $bpuPtr, [ifuPtr] $ifuPtr, [ifuWbPtr] $ifuWbPtr [commPtr] $commPtr\n")
1409  XSDebug(true.B, p"[in] v:${io.fromBpu.resp.valid} r:${io.fromBpu.resp.ready} " +
1410    p"[out] v:${io.toIfu.req.valid} r:${io.toIfu.req.ready}\n")
1411  XSDebug(do_commit, p"[deq info] cfiIndex: $commit_cfi, $commit_pc_bundle, target: ${Hexadecimal(commit_target)}\n")
1412
1413  //   def ubtbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1414  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1415  //       case (((valid, pd), ans), taken) =>
1416  //       Mux(valid && pd.isBr,
1417  //         isWrong ^ Mux(ans.hit.asBool,
1418  //           Mux(ans.taken.asBool, taken && ans.target === commitEntry.target,
1419  //           !taken),
1420  //         !taken),
1421  //       false.B)
1422  //     }
1423  //   }
1424
1425  //   def btbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1426  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1427  //       case (((valid, pd), ans), taken) =>
1428  //       Mux(valid && pd.isBr,
1429  //         isWrong ^ Mux(ans.hit.asBool,
1430  //           Mux(ans.taken.asBool, taken && ans.target === commitEntry.target,
1431  //           !taken),
1432  //         !taken),
1433  //       false.B)
1434  //     }
1435  //   }
1436
1437  //   def tageCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1438  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1439  //       case (((valid, pd), ans), taken) =>
1440  //       Mux(valid && pd.isBr,
1441  //         isWrong ^ (ans.taken.asBool === taken),
1442  //       false.B)
1443  //     }
1444  //   }
1445
1446  //   def loopCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1447  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1448  //       case (((valid, pd), ans), taken) =>
1449  //       Mux(valid && (pd.isBr) && ans.hit.asBool,
1450  //         isWrong ^ (!taken),
1451  //           false.B)
1452  //     }
1453  //   }
1454
1455  //   def rasCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1456  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1457  //       case (((valid, pd), ans), taken) =>
1458  //       Mux(valid && pd.isRet.asBool /*&& taken*/ && ans.hit.asBool,
1459  //         isWrong ^ (ans.target === commitEntry.target),
1460  //           false.B)
1461  //     }
1462  //   }
1463
1464  //   val ubtbRights = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), false.B)
1465  //   val ubtbWrongs = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), true.B)
1466  //   // btb and ubtb pred jal and jalr as well
1467  //   val btbRights = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), false.B)
1468  //   val btbWrongs = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), true.B)
1469  //   val tageRights = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), false.B)
1470  //   val tageWrongs = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), true.B)
1471
1472  //   val loopRights = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), false.B)
1473  //   val loopWrongs = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), true.B)
1474
1475  //   val rasRights = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), false.B)
1476  //   val rasWrongs = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), true.B)
1477
1478  val perfEvents = Seq(
1479    ("bpu_s2_redirect        ", bpu_s2_redirect                                                             ),
1480    ("bpu_s3_redirect        ", bpu_s3_redirect                                                             ),
1481    ("bpu_to_ftq_stall       ", enq.valid && ~enq.ready                                                     ),
1482    ("mispredictRedirect     ", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level),
1483    ("replayRedirect         ", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level)  ),
1484    ("predecodeRedirect      ", fromIfuRedirect.valid                                                       ),
1485    ("to_ifu_bubble          ", io.toIfu.req.ready && !io.toIfu.req.valid                                   ),
1486    ("from_bpu_real_bubble   ", !enq.valid && enq.ready && allowBpuIn                                       ),
1487    ("BpInstr                ", PopCount(mbpInstrs)                                                         ),
1488    ("BpBInstr               ", PopCount(mbpBRights | mbpBWrongs)                                           ),
1489    ("BpRight                ", PopCount(mbpRights)                                                         ),
1490    ("BpWrong                ", PopCount(mbpWrongs)                                                         ),
1491    ("BpBRight               ", PopCount(mbpBRights)                                                        ),
1492    ("BpBWrong               ", PopCount(mbpBWrongs)                                                        ),
1493    ("BpJRight               ", PopCount(mbpJRights)                                                        ),
1494    ("BpJWrong               ", PopCount(mbpJWrongs)                                                        ),
1495    ("BpIRight               ", PopCount(mbpIRights)                                                        ),
1496    ("BpIWrong               ", PopCount(mbpIWrongs)                                                        ),
1497    ("BpCRight               ", PopCount(mbpCRights)                                                        ),
1498    ("BpCWrong               ", PopCount(mbpCWrongs)                                                        ),
1499    ("BpRRight               ", PopCount(mbpRRights)                                                        ),
1500    ("BpRWrong               ", PopCount(mbpRWrongs)                                                        ),
1501    ("ftb_false_hit          ", PopCount(ftb_false_hit)                                                     ),
1502    ("ftb_hit                ", PopCount(ftb_hit)                                                           ),
1503  )
1504  generatePerfEvent()
1505}