xref: /XiangShan/src/main/scala/xiangshan/frontend/NewFtq.scala (revision f4ebc4b283907051c39caf1c39037ec934473d7f)
1/***************************************************************************************
2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
3* Copyright (c) 2020-2021 Peng Cheng Laboratory
4*
5* XiangShan is licensed under Mulan PSL v2.
6* You can use this software according to the terms and conditions of the Mulan PSL v2.
7* You may obtain a copy of Mulan PSL v2 at:
8*          http://license.coscl.org.cn/MulanPSL2
9*
10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
13*
14* See the Mulan PSL v2 for more details.
15***************************************************************************************/
16
17package xiangshan.frontend
18
19import chipsalliance.rocketchip.config.Parameters
20import chisel3._
21import chisel3.util._
22import utils._
23import xiangshan._
24import xiangshan.frontend.icache._
25import xiangshan.backend.CtrlToFtqIO
26
27class FtqPtr(implicit p: Parameters) extends CircularQueuePtr[FtqPtr](
28  p => p(XSCoreParamsKey).FtqSize
29){
30  override def cloneType = (new FtqPtr).asInstanceOf[this.type]
31}
32
33object FtqPtr {
34  def apply(f: Bool, v: UInt)(implicit p: Parameters): FtqPtr = {
35    val ptr = Wire(new FtqPtr)
36    ptr.flag := f
37    ptr.value := v
38    ptr
39  }
40  def inverse(ptr: FtqPtr)(implicit p: Parameters): FtqPtr = {
41    apply(!ptr.flag, ptr.value)
42  }
43}
44
45class FtqNRSRAM[T <: Data](gen: T, numRead: Int)(implicit p: Parameters) extends XSModule {
46
47  val io = IO(new Bundle() {
48    val raddr = Input(Vec(numRead, UInt(log2Up(FtqSize).W)))
49    val ren = Input(Vec(numRead, Bool()))
50    val rdata = Output(Vec(numRead, gen))
51    val waddr = Input(UInt(log2Up(FtqSize).W))
52    val wen = Input(Bool())
53    val wdata = Input(gen)
54  })
55
56  for(i <- 0 until numRead){
57    val sram = Module(new SRAMTemplate(gen, FtqSize))
58    sram.io.r.req.valid := io.ren(i)
59    sram.io.r.req.bits.setIdx := io.raddr(i)
60    io.rdata(i) := sram.io.r.resp.data(0)
61    sram.io.w.req.valid := io.wen
62    sram.io.w.req.bits.setIdx := io.waddr
63    sram.io.w.req.bits.data := VecInit(io.wdata)
64  }
65
66}
67
68class Ftq_RF_Components(implicit p: Parameters) extends XSBundle with BPUUtils {
69  val startAddr = UInt(VAddrBits.W)
70  val nextLineAddr = UInt(VAddrBits.W)
71  val isNextMask = Vec(PredictWidth, Bool())
72  val fallThruError = Bool()
73  // val carry = Bool()
74  def getPc(offset: UInt) = {
75    def getHigher(pc: UInt) = pc(VAddrBits-1, log2Ceil(PredictWidth)+instOffsetBits+1)
76    def getOffset(pc: UInt) = pc(log2Ceil(PredictWidth)+instOffsetBits, instOffsetBits)
77    Cat(getHigher(Mux(isNextMask(offset) && startAddr(log2Ceil(PredictWidth)+instOffsetBits), nextLineAddr, startAddr)),
78        getOffset(startAddr)+offset, 0.U(instOffsetBits.W))
79  }
80  def fromBranchPrediction(resp: BranchPredictionBundle) = {
81    def carryPos(addr: UInt) = addr(instOffsetBits+log2Ceil(PredictWidth)+1)
82    this.startAddr := resp.pc
83    this.nextLineAddr := resp.pc + (FetchWidth * 4 * 2).U // may be broken on other configs
84    this.isNextMask := VecInit((0 until PredictWidth).map(i =>
85      (resp.pc(log2Ceil(PredictWidth), 1) +& i.U)(log2Ceil(PredictWidth)).asBool()
86    ))
87    this.fallThruError := resp.fallThruError
88    this
89  }
90  override def toPrintable: Printable = {
91    p"startAddr:${Hexadecimal(startAddr)}"
92  }
93}
94
95class Ftq_pd_Entry(implicit p: Parameters) extends XSBundle {
96  val brMask = Vec(PredictWidth, Bool())
97  val jmpInfo = ValidUndirectioned(Vec(3, Bool()))
98  val jmpOffset = UInt(log2Ceil(PredictWidth).W)
99  val jalTarget = UInt(VAddrBits.W)
100  val rvcMask = Vec(PredictWidth, Bool())
101  def hasJal  = jmpInfo.valid && !jmpInfo.bits(0)
102  def hasJalr = jmpInfo.valid && jmpInfo.bits(0)
103  def hasCall = jmpInfo.valid && jmpInfo.bits(1)
104  def hasRet  = jmpInfo.valid && jmpInfo.bits(2)
105
106  def fromPdWb(pdWb: PredecodeWritebackBundle) = {
107    val pds = pdWb.pd
108    this.brMask := VecInit(pds.map(pd => pd.isBr && pd.valid))
109    this.jmpInfo.valid := VecInit(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid)).asUInt.orR
110    this.jmpInfo.bits := ParallelPriorityMux(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid),
111                                             pds.map(pd => VecInit(pd.isJalr, pd.isCall, pd.isRet)))
112    this.jmpOffset := ParallelPriorityEncoder(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid))
113    this.rvcMask := VecInit(pds.map(pd => pd.isRVC))
114    this.jalTarget := pdWb.jalTarget
115  }
116
117  def toPd(offset: UInt) = {
118    require(offset.getWidth == log2Ceil(PredictWidth))
119    val pd = Wire(new PreDecodeInfo)
120    pd.valid := true.B
121    pd.isRVC := rvcMask(offset)
122    val isBr = brMask(offset)
123    val isJalr = offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(0)
124    pd.brType := Cat(offset === jmpOffset && jmpInfo.valid, isJalr || isBr)
125    pd.isCall := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(1)
126    pd.isRet  := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(2)
127    pd
128  }
129}
130
131
132
133class Ftq_Redirect_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst {
134  val rasSp = UInt(log2Ceil(RasSize).W)
135  val rasEntry = new RASEntry
136  // val specCnt = Vec(numBr, UInt(10.W))
137  // val ghist = new ShiftingGlobalHistory
138  val folded_hist = new AllFoldedHistories(foldedGHistInfos)
139  val afhob = new AllAheadFoldedHistoryOldestBits(foldedGHistInfos)
140  val lastBrNumOH = UInt((numBr+1).W)
141
142  val histPtr = new CGHPtr
143
144  def fromBranchPrediction(resp: BranchPredictionBundle) = {
145    assert(!resp.is_minimal)
146    this.rasSp := resp.rasSp
147    this.rasEntry := resp.rasTop
148    this.folded_hist := resp.folded_hist
149    this.afhob := resp.afhob
150    this.lastBrNumOH := resp.lastBrNumOH
151    this.histPtr := resp.histPtr
152    this
153  }
154}
155
156class Ftq_1R_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst {
157  val meta = UInt(MaxMetaLength.W)
158}
159
160class Ftq_Pred_Info(implicit p: Parameters) extends XSBundle {
161  val target = UInt(VAddrBits.W)
162  val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))
163}
164
165// class FtqEntry(implicit p: Parameters) extends XSBundle with HasBPUConst {
166//   val startAddr = UInt(VAddrBits.W)
167//   val fallThruAddr = UInt(VAddrBits.W)
168//   val isNextMask = Vec(PredictWidth, Bool())
169
170//   val meta = UInt(MaxMetaLength.W)
171
172//   val rasSp = UInt(log2Ceil(RasSize).W)
173//   val rasEntry = new RASEntry
174//   val hist = new ShiftingGlobalHistory
175//   val specCnt = Vec(numBr, UInt(10.W))
176
177//   val valids = Vec(PredictWidth, Bool())
178//   val brMask = Vec(PredictWidth, Bool())
179//   // isJalr, isCall, isRet
180//   val jmpInfo = ValidUndirectioned(Vec(3, Bool()))
181//   val jmpOffset = UInt(log2Ceil(PredictWidth).W)
182
183//   val mispredVec = Vec(PredictWidth, Bool())
184//   val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))
185//   val target = UInt(VAddrBits.W)
186// }
187
188class FtqRead[T <: Data](private val gen: T)(implicit p: Parameters) extends XSBundle {
189  val ptr = Output(new FtqPtr)
190  val offset = Output(UInt(log2Ceil(PredictWidth).W))
191  val data = Input(gen)
192  def apply(ptr: FtqPtr, offset: UInt) = {
193    this.ptr := ptr
194    this.offset := offset
195    this.data
196  }
197  override def cloneType = (new FtqRead(gen)).asInstanceOf[this.type]
198}
199
200
201class FtqToBpuIO(implicit p: Parameters) extends XSBundle {
202  val redirect = Valid(new BranchPredictionRedirect)
203  val update = Valid(new BranchPredictionUpdate)
204  val enq_ptr = Output(new FtqPtr)
205}
206
207class FtqToIfuIO(implicit p: Parameters) extends XSBundle with HasCircularQueuePtrHelper {
208  val req = Decoupled(new FetchRequestBundle)
209  val redirect = Valid(new Redirect)
210  val flushFromBpu = new Bundle {
211    // when ifu pipeline is not stalled,
212    // a packet from bpu s3 can reach f1 at most
213    val s2 = Valid(new FtqPtr)
214    val s3 = Valid(new FtqPtr)
215    def shouldFlushBy(src: Valid[FtqPtr], idx_to_flush: FtqPtr) = {
216      src.valid && !isAfter(src.bits, idx_to_flush)
217    }
218    def shouldFlushByStage2(idx: FtqPtr) = shouldFlushBy(s2, idx)
219    def shouldFlushByStage3(idx: FtqPtr) = shouldFlushBy(s3, idx)
220  }
221}
222
223trait HasBackendRedirectInfo extends HasXSParameter {
224  def numRedirect = exuParameters.JmpCnt + exuParameters.AluCnt + 1
225  def isLoadReplay(r: Valid[Redirect]) = r.bits.flushItself()
226}
227
228class FtqToCtrlIO(implicit p: Parameters) extends XSBundle with HasBackendRedirectInfo {
229  val pc_reads = Vec(1 + numRedirect + 1 + 1, Flipped(new FtqRead(UInt(VAddrBits.W))))
230  val target_read = Flipped(new FtqRead(UInt(VAddrBits.W)))
231  def getJumpPcRead = pc_reads.head
232  def getRedirectPcRead = VecInit(pc_reads.tail.dropRight(2))
233  def getMemPredPcRead = pc_reads.init.last
234  def getRobFlushPcRead = pc_reads.last
235}
236
237
238class FTBEntryGen(implicit p: Parameters) extends XSModule with HasBackendRedirectInfo with HasBPUParameter {
239  val io = IO(new Bundle {
240    val start_addr = Input(UInt(VAddrBits.W))
241    val old_entry = Input(new FTBEntry)
242    val pd = Input(new Ftq_pd_Entry)
243    val cfiIndex = Flipped(Valid(UInt(log2Ceil(PredictWidth).W)))
244    val target = Input(UInt(VAddrBits.W))
245    val hit = Input(Bool())
246    val mispredict_vec = Input(Vec(PredictWidth, Bool()))
247
248    val new_entry = Output(new FTBEntry)
249    val new_br_insert_pos = Output(Vec(numBr, Bool()))
250    val taken_mask = Output(Vec(numBr, Bool()))
251    val mispred_mask = Output(Vec(numBr+1, Bool()))
252
253    // for perf counters
254    val is_init_entry = Output(Bool())
255    val is_old_entry = Output(Bool())
256    val is_new_br = Output(Bool())
257    val is_jalr_target_modified = Output(Bool())
258    val is_always_taken_modified = Output(Bool())
259    val is_br_full = Output(Bool())
260  })
261
262  // no mispredictions detected at predecode
263  val hit = io.hit
264  val pd = io.pd
265
266  val init_entry = WireInit(0.U.asTypeOf(new FTBEntry))
267
268
269  val cfi_is_br = pd.brMask(io.cfiIndex.bits) && io.cfiIndex.valid
270  val entry_has_jmp = pd.jmpInfo.valid
271  val new_jmp_is_jal  = entry_has_jmp && !pd.jmpInfo.bits(0) && io.cfiIndex.valid
272  val new_jmp_is_jalr = entry_has_jmp &&  pd.jmpInfo.bits(0) && io.cfiIndex.valid
273  val new_jmp_is_call = entry_has_jmp &&  pd.jmpInfo.bits(1) && io.cfiIndex.valid
274  val new_jmp_is_ret  = entry_has_jmp &&  pd.jmpInfo.bits(2) && io.cfiIndex.valid
275  val last_jmp_rvi = entry_has_jmp && pd.jmpOffset === (PredictWidth-1).U && !pd.rvcMask.last
276  // val last_br_rvi = cfi_is_br && io.cfiIndex.bits === (PredictWidth-1).U && !pd.rvcMask.last
277
278  val cfi_is_jal = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jal
279  val cfi_is_jalr = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jalr
280
281  def carryPos = log2Ceil(PredictWidth)+instOffsetBits
282  def getLower(pc: UInt) = pc(carryPos-1, instOffsetBits)
283  // if not hit, establish a new entry
284  init_entry.valid := true.B
285  // tag is left for ftb to assign
286
287  // case br
288  val init_br_slot = init_entry.getSlotForBr(0)
289  when (cfi_is_br) {
290    init_br_slot.valid := true.B
291    init_br_slot.offset := io.cfiIndex.bits
292    init_br_slot.setLowerStatByTarget(io.start_addr, io.target, numBr == 1)
293    init_entry.always_taken(0) := true.B // set to always taken on init
294  }
295
296  // case jmp
297  when (entry_has_jmp) {
298    init_entry.tailSlot.offset := pd.jmpOffset
299    init_entry.tailSlot.valid := new_jmp_is_jal || new_jmp_is_jalr
300    init_entry.tailSlot.setLowerStatByTarget(io.start_addr, Mux(cfi_is_jalr, io.target, pd.jalTarget), isShare=false)
301  }
302
303  val jmpPft = getLower(io.start_addr) +& pd.jmpOffset +& Mux(pd.rvcMask(pd.jmpOffset), 1.U, 2.U)
304  init_entry.pftAddr := Mux(entry_has_jmp && !last_jmp_rvi, jmpPft, getLower(io.start_addr))
305  init_entry.carry   := Mux(entry_has_jmp && !last_jmp_rvi, jmpPft(carryPos-instOffsetBits), true.B)
306  init_entry.isJalr := new_jmp_is_jalr
307  init_entry.isCall := new_jmp_is_call
308  init_entry.isRet  := new_jmp_is_ret
309  // that means fall thru points to the middle of an inst
310  init_entry.last_may_be_rvi_call := io.cfiIndex.bits === (PredictWidth-1).U && !pd.rvcMask(pd.jmpOffset)
311
312  // if hit, check whether a new cfi(only br is possible) is detected
313  val oe = io.old_entry
314  val br_recorded_vec = oe.getBrRecordedVec(io.cfiIndex.bits)
315  val br_recorded = br_recorded_vec.asUInt.orR
316  val is_new_br = cfi_is_br && !br_recorded
317  val new_br_offset = io.cfiIndex.bits
318  // vec(i) means new br will be inserted BEFORE old br(i)
319  val allBrSlotsVec = oe.allSlotsForBr
320  val new_br_insert_onehot = VecInit((0 until numBr).map{
321    i => i match {
322      case 0 =>
323        !allBrSlotsVec(0).valid || new_br_offset < allBrSlotsVec(0).offset
324      case idx =>
325        allBrSlotsVec(idx-1).valid && new_br_offset > allBrSlotsVec(idx-1).offset &&
326        (!allBrSlotsVec(idx).valid || new_br_offset < allBrSlotsVec(idx).offset)
327    }
328  })
329
330  val old_entry_modified = WireInit(io.old_entry)
331  for (i <- 0 until numBr) {
332    val slot = old_entry_modified.allSlotsForBr(i)
333    when (new_br_insert_onehot(i)) {
334      slot.valid := true.B
335      slot.offset := new_br_offset
336      slot.setLowerStatByTarget(io.start_addr, io.target, i == numBr-1)
337      old_entry_modified.always_taken(i) := true.B
338    }.elsewhen (new_br_offset > oe.allSlotsForBr(i).offset) {
339      old_entry_modified.always_taken(i) := false.B
340      // all other fields remain unchanged
341    }.otherwise {
342      // case i == 0, remain unchanged
343      if (i != 0) {
344        val noNeedToMoveFromFormerSlot = (i == numBr-1).B && !oe.brSlots.last.valid
345        when (!noNeedToMoveFromFormerSlot) {
346          slot.fromAnotherSlot(oe.allSlotsForBr(i-1))
347          old_entry_modified.always_taken(i) := oe.always_taken(i)
348        }
349      }
350    }
351  }
352
353  // two circumstances:
354  // 1. oe: | br | j  |, new br should be in front of j, thus addr of j should be new pft
355  // 2. oe: | br | br |, new br could be anywhere between, thus new pft is the addr of either
356  //        the previous last br or the new br
357  val may_have_to_replace = oe.noEmptySlotForNewBr
358  val pft_need_to_change = is_new_br && may_have_to_replace
359  // it should either be the given last br or the new br
360  when (pft_need_to_change) {
361    val new_pft_offset =
362      Mux(!new_br_insert_onehot.asUInt.orR,
363        new_br_offset, oe.allSlotsForBr.last.offset)
364
365    // set jmp to invalid
366    old_entry_modified.pftAddr := getLower(io.start_addr) + new_pft_offset
367    old_entry_modified.carry := (getLower(io.start_addr) +& new_pft_offset).head(1).asBool
368    old_entry_modified.last_may_be_rvi_call := false.B
369    old_entry_modified.isCall := false.B
370    old_entry_modified.isRet := false.B
371    old_entry_modified.isJalr := false.B
372  }
373
374  val old_entry_jmp_target_modified = WireInit(oe)
375  val old_target = oe.tailSlot.getTarget(io.start_addr) // may be wrong because we store only 20 lowest bits
376  val old_tail_is_jmp = !oe.tailSlot.sharing
377  val jalr_target_modified = cfi_is_jalr && (old_target =/= io.target) && old_tail_is_jmp // TODO: pass full jalr target
378  when (jalr_target_modified) {
379    old_entry_jmp_target_modified.setByJmpTarget(io.start_addr, io.target)
380    old_entry_jmp_target_modified.always_taken := 0.U.asTypeOf(Vec(numBr, Bool()))
381  }
382
383  val old_entry_always_taken = WireInit(oe)
384  val always_taken_modified_vec = Wire(Vec(numBr, Bool())) // whether modified or not
385  for (i <- 0 until numBr) {
386    old_entry_always_taken.always_taken(i) :=
387      oe.always_taken(i) && io.cfiIndex.valid && oe.brValids(i) && io.cfiIndex.bits === oe.brOffset(i)
388    always_taken_modified_vec(i) := oe.always_taken(i) && !old_entry_always_taken.always_taken(i)
389  }
390  val always_taken_modified = always_taken_modified_vec.reduce(_||_)
391
392
393
394  val derived_from_old_entry =
395    Mux(is_new_br, old_entry_modified,
396      Mux(jalr_target_modified, old_entry_jmp_target_modified, old_entry_always_taken))
397
398
399  io.new_entry := Mux(!hit, init_entry, derived_from_old_entry)
400
401  io.new_br_insert_pos := new_br_insert_onehot
402  io.taken_mask := VecInit((io.new_entry.brOffset zip io.new_entry.brValids).map{
403    case (off, v) => io.cfiIndex.bits === off && io.cfiIndex.valid && v
404  })
405  for (i <- 0 until numBr) {
406    io.mispred_mask(i) := io.new_entry.brValids(i) && io.mispredict_vec(io.new_entry.brOffset(i))
407  }
408  io.mispred_mask.last := io.new_entry.jmpValid && io.mispredict_vec(pd.jmpOffset)
409
410  // for perf counters
411  io.is_init_entry := !hit
412  io.is_old_entry := hit && !is_new_br && !jalr_target_modified && !always_taken_modified
413  io.is_new_br := hit && is_new_br
414  io.is_jalr_target_modified := hit && jalr_target_modified
415  io.is_always_taken_modified := hit && always_taken_modified
416  io.is_br_full := hit && is_new_br && may_have_to_replace
417}
418
419class Ftq(implicit p: Parameters) extends XSModule with HasCircularQueuePtrHelper
420  with HasBackendRedirectInfo with BPUUtils with HasBPUConst with HasPerfEvents
421  with HasICacheParameters{
422  val io = IO(new Bundle {
423    val fromBpu = Flipped(new BpuToFtqIO)
424    val fromIfu = Flipped(new IfuToFtqIO)
425    val fromBackend = Flipped(new CtrlToFtqIO)
426
427    val toBpu = new FtqToBpuIO
428    val toIfu = new FtqToIfuIO
429    val toBackend = new FtqToCtrlIO
430
431    val toPrefetch = new FtqPrefechBundle
432
433    val bpuInfo = new Bundle {
434      val bpRight = Output(UInt(XLEN.W))
435      val bpWrong = Output(UInt(XLEN.W))
436    }
437  })
438  io.bpuInfo := DontCare
439
440  val backendRedirect = io.fromBackend.redirect
441  val backendRedirectReg = RegNext(io.fromBackend.redirect)
442
443  val stage2Flush = backendRedirect.valid
444  val backendFlush = stage2Flush || RegNext(stage2Flush)
445  val ifuFlush = Wire(Bool())
446
447  val flush = stage2Flush || RegNext(stage2Flush)
448
449  val allowBpuIn, allowToIfu = WireInit(false.B)
450  val flushToIfu = !allowToIfu
451  allowBpuIn := !ifuFlush && !backendRedirect.valid && !backendRedirectReg.valid
452  allowToIfu := !ifuFlush && !backendRedirect.valid && !backendRedirectReg.valid
453
454  val bpuPtr, ifuPtr, ifuWbPtr, commPtr = RegInit(FtqPtr(false.B, 0.U))
455  val validEntries = distanceBetween(bpuPtr, commPtr)
456
457  // **********************************************************************
458  // **************************** enq from bpu ****************************
459  // **********************************************************************
460  val new_entry_ready = validEntries < FtqSize.U
461  io.fromBpu.resp.ready := new_entry_ready
462
463  val bpu_s2_resp = io.fromBpu.resp.bits.s2
464  val bpu_s3_resp = io.fromBpu.resp.bits.s3
465  val bpu_s2_redirect = bpu_s2_resp.valid && bpu_s2_resp.hasRedirect
466  val bpu_s3_redirect = bpu_s3_resp.valid && bpu_s3_resp.hasRedirect
467
468  io.toBpu.enq_ptr := bpuPtr
469  val enq_fire = io.fromBpu.resp.fire() && allowBpuIn // from bpu s1
470  val bpu_in_fire = (io.fromBpu.resp.fire() || bpu_s2_redirect || bpu_s3_redirect) && allowBpuIn
471
472  val bpu_in_resp = io.fromBpu.resp.bits.selectedResp
473  val bpu_in_stage = io.fromBpu.resp.bits.selectedRespIdx
474  val bpu_in_resp_ptr = Mux(bpu_in_stage === BP_S1, bpuPtr, bpu_in_resp.ftq_idx)
475  val bpu_in_resp_idx = bpu_in_resp_ptr.value
476
477  // read ports:                            jumpPc + redirects + loadPred + robFlush + ifuReq1 + ifuReq2 + commitUpdate
478  val ftq_pc_mem = Module(new SyncDataModuleTemplate(new Ftq_RF_Components, FtqSize, 1+numRedirect+2+1+1+1, 1))
479  // resp from uBTB
480  ftq_pc_mem.io.wen(0) := bpu_in_fire
481  ftq_pc_mem.io.waddr(0) := bpu_in_resp_idx
482  ftq_pc_mem.io.wdata(0).fromBranchPrediction(bpu_in_resp)
483
484  //                                                            ifuRedirect + backendRedirect + commit
485  val ftq_redirect_sram = Module(new FtqNRSRAM(new Ftq_Redirect_SRAMEntry, 1+1+1))
486  // these info is intended to enq at the last stage of bpu
487  ftq_redirect_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid
488  ftq_redirect_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value
489  ftq_redirect_sram.io.wdata.fromBranchPrediction(io.fromBpu.resp.bits.lastStage)
490  println(f"ftq redirect SRAM: entry ${ftq_redirect_sram.io.wdata.getWidth} * ${FtqSize} * 3")
491  println(f"ftq redirect SRAM: ahead fh ${ftq_redirect_sram.io.wdata.afhob.getWidth} * ${FtqSize} * 3")
492
493  val ftq_meta_1r_sram = Module(new FtqNRSRAM(new Ftq_1R_SRAMEntry, 1))
494  // these info is intended to enq at the last stage of bpu
495  ftq_meta_1r_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid
496  ftq_meta_1r_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value
497  ftq_meta_1r_sram.io.wdata.meta := io.fromBpu.resp.bits.meta
498  //                                                            ifuRedirect + backendRedirect + commit
499  val ftb_entry_mem = Module(new SyncDataModuleTemplate(new FTBEntry, FtqSize, 1+1+1, 1))
500  ftb_entry_mem.io.wen(0) := io.fromBpu.resp.bits.lastStage.valid
501  ftb_entry_mem.io.waddr(0) := io.fromBpu.resp.bits.lastStage.ftq_idx.value
502  ftb_entry_mem.io.wdata(0) := io.fromBpu.resp.bits.lastStage.ftb_entry
503
504
505  // multi-write
506  val update_target = Reg(Vec(FtqSize, UInt(VAddrBits.W))) // could be taken target or fallThrough
507  val cfiIndex_vec = Reg(Vec(FtqSize, ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))))
508  val mispredict_vec = Reg(Vec(FtqSize, Vec(PredictWidth, Bool())))
509  val pred_stage = Reg(Vec(FtqSize, UInt(2.W)))
510
511  val c_invalid :: c_valid :: c_commited :: Nil = Enum(3)
512  val commitStateQueue = RegInit(VecInit(Seq.fill(FtqSize) {
513    VecInit(Seq.fill(PredictWidth)(c_invalid))
514  }))
515
516  val f_to_send :: f_sent :: Nil = Enum(2)
517  val entry_fetch_status = RegInit(VecInit(Seq.fill(FtqSize)(f_sent)))
518
519  val h_not_hit :: h_false_hit :: h_hit :: Nil = Enum(3)
520  val entry_hit_status = RegInit(VecInit(Seq.fill(FtqSize)(h_not_hit)))
521
522
523  when (bpu_in_fire) {
524    entry_fetch_status(bpu_in_resp_idx) := f_to_send
525    commitStateQueue(bpu_in_resp_idx) := VecInit(Seq.fill(PredictWidth)(c_invalid))
526    cfiIndex_vec(bpu_in_resp_idx) := bpu_in_resp.cfiIndex
527    mispredict_vec(bpu_in_resp_idx) := WireInit(VecInit(Seq.fill(PredictWidth)(false.B)))
528    update_target(bpu_in_resp_idx) := bpu_in_resp.getTarget
529    pred_stage(bpu_in_resp_idx) := bpu_in_stage
530  }
531
532  bpuPtr := bpuPtr + enq_fire
533  ifuPtr := ifuPtr + (io.toIfu.req.fire && allowToIfu)
534
535  // only use ftb result to assign hit status
536  when (bpu_s2_resp.valid) {
537    entry_hit_status(bpu_s2_resp.ftq_idx.value) := Mux(bpu_s2_resp.full_pred.hit, h_hit, h_not_hit)
538  }
539
540
541  io.toIfu.flushFromBpu.s2.valid := bpu_s2_redirect
542  io.toIfu.flushFromBpu.s2.bits := bpu_s2_resp.ftq_idx
543  when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect) {
544    bpuPtr := bpu_s2_resp.ftq_idx + 1.U
545    // only when ifuPtr runs ahead of bpu s2 resp should we recover it
546    when (!isBefore(ifuPtr, bpu_s2_resp.ftq_idx)) {
547      ifuPtr := bpu_s2_resp.ftq_idx
548    }
549  }
550
551  io.toIfu.flushFromBpu.s3.valid := bpu_s3_redirect
552  io.toIfu.flushFromBpu.s3.bits := bpu_s3_resp.ftq_idx
553  when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect) {
554    bpuPtr := bpu_s3_resp.ftq_idx + 1.U
555    // only when ifuPtr runs ahead of bpu s2 resp should we recover it
556    when (!isBefore(ifuPtr, bpu_s3_resp.ftq_idx)) {
557      ifuPtr := bpu_s3_resp.ftq_idx
558    }
559  }
560
561  XSError(isBefore(bpuPtr, ifuPtr) && !isFull(bpuPtr, ifuPtr), "\nifuPtr is before bpuPtr!\n")
562
563  // ****************************************************************
564  // **************************** to ifu ****************************
565  // ****************************************************************
566  val bpu_in_bypass_buf = RegEnable(ftq_pc_mem.io.wdata(0), enable=bpu_in_fire)
567  val bpu_in_bypass_ptr = RegNext(bpu_in_resp_ptr)
568  val last_cycle_bpu_in = RegNext(bpu_in_fire)
569  val last_cycle_to_ifu_fire = RegNext(io.toIfu.req.fire)
570
571  // read pc and target
572  ftq_pc_mem.io.raddr.init.init.last := ifuPtr.value
573  ftq_pc_mem.io.raddr.init.last := (ifuPtr+1.U).value
574
575  io.toIfu.req.valid := entry_fetch_status(ifuPtr.value) === f_to_send && ifuPtr =/= bpuPtr
576  io.toIfu.req.bits.ftqIdx := ifuPtr
577  io.toIfu.req.bits.nextStartAddr := update_target(ifuPtr.value)
578  io.toIfu.req.bits.ftqOffset := cfiIndex_vec(ifuPtr.value)
579
580  val toIfuPcBundle = Wire(new Ftq_RF_Components)
581
582  when (last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr) {
583    toIfuPcBundle := bpu_in_bypass_buf
584  }.elsewhen (last_cycle_to_ifu_fire) {
585    toIfuPcBundle := ftq_pc_mem.io.rdata.init.last
586  }.otherwise {
587    toIfuPcBundle := ftq_pc_mem.io.rdata.init.init.last
588  }
589
590  io.toIfu.req.bits.fromFtqPcBundle(toIfuPcBundle)
591
592  // when fall through is smaller in value than start address, there must be a false hit
593  when (toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit) {
594    when (io.toIfu.req.fire &&
595      !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr) &&
596      !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr)
597    ) {
598      entry_hit_status(ifuPtr.value) := h_false_hit
599      // XSError(true.B, "FTB false hit by fallThroughError, startAddr: %x, fallTHru: %x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr)
600    }
601    XSDebug(true.B, "fallThruError! start:%x, fallThru:%x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr)
602  }
603
604  XSPerfAccumulate(f"fall_through_error_to_ifu", toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit &&
605    io.toIfu.req.fire && !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr) && !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr))
606
607  val ifu_req_should_be_flushed =
608    io.toIfu.flushFromBpu.shouldFlushByStage2(io.toIfu.req.bits.ftqIdx) ||
609    io.toIfu.flushFromBpu.shouldFlushByStage3(io.toIfu.req.bits.ftqIdx)
610
611    when (io.toIfu.req.fire && !ifu_req_should_be_flushed) {
612      entry_fetch_status(ifuPtr.value) := f_sent
613    }
614
615  // *********************************************************************
616  // **************************** wb from ifu ****************************
617  // *********************************************************************
618  val pdWb = io.fromIfu.pdWb
619  val pds = pdWb.bits.pd
620  val ifu_wb_valid = pdWb.valid
621  val ifu_wb_idx = pdWb.bits.ftqIdx.value
622  // read ports:                                                         commit update
623  val ftq_pd_mem = Module(new SyncDataModuleTemplate(new Ftq_pd_Entry, FtqSize, 1, 1))
624  ftq_pd_mem.io.wen(0) := ifu_wb_valid
625  ftq_pd_mem.io.waddr(0) := pdWb.bits.ftqIdx.value
626  ftq_pd_mem.io.wdata(0).fromPdWb(pdWb.bits)
627
628  val hit_pd_valid = entry_hit_status(ifu_wb_idx) === h_hit && ifu_wb_valid
629  val hit_pd_mispred = hit_pd_valid && pdWb.bits.misOffset.valid
630  val hit_pd_mispred_reg = RegNext(hit_pd_mispred, init=false.B)
631  val pd_reg       = RegEnable(pds,             enable = pdWb.valid)
632  val start_pc_reg = RegEnable(pdWb.bits.pc(0), enable = pdWb.valid)
633  val wb_idx_reg   = RegEnable(ifu_wb_idx,      enable = pdWb.valid)
634
635  when (ifu_wb_valid) {
636    val comm_stq_wen = VecInit(pds.map(_.valid).zip(pdWb.bits.instrRange).map{
637      case (v, inRange) => v && inRange
638    })
639    (commitStateQueue(ifu_wb_idx) zip comm_stq_wen).map{
640      case (qe, v) => when (v) { qe := c_valid }
641    }
642  }
643
644  ifuWbPtr := ifuWbPtr + ifu_wb_valid
645
646  ftb_entry_mem.io.raddr.head := ifu_wb_idx
647  val has_false_hit = WireInit(false.B)
648  when (RegNext(hit_pd_valid)) {
649    // check for false hit
650    val pred_ftb_entry = ftb_entry_mem.io.rdata.head
651    val brSlots = pred_ftb_entry.brSlots
652    val tailSlot = pred_ftb_entry.tailSlot
653    // we check cfis that bpu predicted
654
655    // bpu predicted branches but denied by predecode
656    val br_false_hit =
657      brSlots.map{
658        s => s.valid && !(pd_reg(s.offset).valid && pd_reg(s.offset).isBr)
659      }.reduce(_||_) ||
660      (tailSlot.valid && pred_ftb_entry.tailSlot.sharing &&
661        !(pd_reg(tailSlot.offset).valid && pd_reg(tailSlot.offset).isBr))
662
663    val jmpOffset = tailSlot.offset
664    val jmp_pd = pd_reg(jmpOffset)
665    val jal_false_hit = pred_ftb_entry.jmpValid &&
666      ((pred_ftb_entry.isJal  && !(jmp_pd.valid && jmp_pd.isJal)) ||
667       (pred_ftb_entry.isJalr && !(jmp_pd.valid && jmp_pd.isJalr)) ||
668       (pred_ftb_entry.isCall && !(jmp_pd.valid && jmp_pd.isCall)) ||
669       (pred_ftb_entry.isRet  && !(jmp_pd.valid && jmp_pd.isRet))
670      )
671
672    has_false_hit := br_false_hit || jal_false_hit || hit_pd_mispred_reg
673    XSDebug(has_false_hit, "FTB false hit by br or jal or hit_pd, startAddr: %x\n", pdWb.bits.pc(0))
674
675    // assert(!has_false_hit)
676  }
677
678  when (has_false_hit) {
679    entry_hit_status(wb_idx_reg) := h_false_hit
680  }
681
682
683  // **********************************************************************
684  // **************************** backend read ****************************
685  // **********************************************************************
686
687  // pc reads
688  for ((req, i) <- io.toBackend.pc_reads.zipWithIndex) {
689    ftq_pc_mem.io.raddr(i) := req.ptr.value
690    req.data := ftq_pc_mem.io.rdata(i).getPc(RegNext(req.offset))
691  }
692  // target read
693  io.toBackend.target_read.data := RegNext(update_target(io.toBackend.target_read.ptr.value))
694
695  // *******************************************************************************
696  // **************************** redirect from backend ****************************
697  // *******************************************************************************
698
699  // redirect read cfiInfo, couples to redirectGen s2
700  ftq_redirect_sram.io.ren.init.last := io.fromBackend.redirect.valid
701  ftq_redirect_sram.io.raddr.init.last := io.fromBackend.redirect.bits.ftqIdx.value
702
703  ftb_entry_mem.io.raddr.init.last := io.fromBackend.redirect.bits.ftqIdx.value
704
705  val stage3CfiInfo = ftq_redirect_sram.io.rdata.init.last
706  val fromBackendRedirect = WireInit(backendRedirectReg)
707  val backendRedirectCfi = fromBackendRedirect.bits.cfiUpdate
708  backendRedirectCfi.fromFtqRedirectSram(stage3CfiInfo)
709
710  val r_ftb_entry = ftb_entry_mem.io.rdata.init.last
711  val r_ftqOffset = fromBackendRedirect.bits.ftqOffset
712
713  when (entry_hit_status(fromBackendRedirect.bits.ftqIdx.value) === h_hit) {
714    backendRedirectCfi.shift := PopCount(r_ftb_entry.getBrMaskByOffset(r_ftqOffset)) +&
715      (backendRedirectCfi.pd.isBr && !r_ftb_entry.brIsSaved(r_ftqOffset) &&
716      !r_ftb_entry.newBrCanNotInsert(r_ftqOffset))
717
718    backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr && (r_ftb_entry.brIsSaved(r_ftqOffset) ||
719        !r_ftb_entry.newBrCanNotInsert(r_ftqOffset))
720  }.otherwise {
721    backendRedirectCfi.shift := (backendRedirectCfi.pd.isBr && backendRedirectCfi.taken).asUInt
722    backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr.asUInt
723  }
724
725
726  // ***************************************************************************
727  // **************************** redirect from ifu ****************************
728  // ***************************************************************************
729  val fromIfuRedirect = WireInit(0.U.asTypeOf(Valid(new Redirect)))
730  fromIfuRedirect.valid := pdWb.valid && pdWb.bits.misOffset.valid && !backendFlush
731  fromIfuRedirect.bits.ftqIdx := pdWb.bits.ftqIdx
732  fromIfuRedirect.bits.ftqOffset := pdWb.bits.misOffset.bits
733  fromIfuRedirect.bits.level := RedirectLevel.flushAfter
734
735  val ifuRedirectCfiUpdate = fromIfuRedirect.bits.cfiUpdate
736  ifuRedirectCfiUpdate.pc := pdWb.bits.pc(pdWb.bits.misOffset.bits)
737  ifuRedirectCfiUpdate.pd := pdWb.bits.pd(pdWb.bits.misOffset.bits)
738  ifuRedirectCfiUpdate.predTaken := cfiIndex_vec(pdWb.bits.ftqIdx.value).valid
739  ifuRedirectCfiUpdate.target := pdWb.bits.target
740  ifuRedirectCfiUpdate.taken := pdWb.bits.cfiOffset.valid
741  ifuRedirectCfiUpdate.isMisPred := pdWb.bits.misOffset.valid
742
743  val ifuRedirectReg = RegNext(fromIfuRedirect, init=0.U.asTypeOf(Valid(new Redirect)))
744  val ifuRedirectToBpu = WireInit(ifuRedirectReg)
745  ifuFlush := fromIfuRedirect.valid || ifuRedirectToBpu.valid
746
747  ftq_redirect_sram.io.ren.head := fromIfuRedirect.valid
748  ftq_redirect_sram.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value
749
750  ftb_entry_mem.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value
751
752  val toBpuCfi = ifuRedirectToBpu.bits.cfiUpdate
753  toBpuCfi.fromFtqRedirectSram(ftq_redirect_sram.io.rdata.head)
754  when (ifuRedirectReg.bits.cfiUpdate.pd.isRet) {
755    toBpuCfi.target := toBpuCfi.rasEntry.retAddr
756  }
757
758  // *********************************************************************
759  // **************************** wb from exu ****************************
760  // *********************************************************************
761
762  def extractRedirectInfo(wb: Valid[Redirect]) = {
763    val ftqIdx = wb.bits.ftqIdx.value
764    val ftqOffset = wb.bits.ftqOffset
765    val taken = wb.bits.cfiUpdate.taken
766    val mispred = wb.bits.cfiUpdate.isMisPred
767    (wb.valid, ftqIdx, ftqOffset, taken, mispred)
768  }
769
770  // fix mispredict entry
771  val lastIsMispredict = RegNext(
772    backendRedirect.valid && backendRedirect.bits.level === RedirectLevel.flushAfter, init = false.B
773  )
774
775  def updateCfiInfo(redirect: Valid[Redirect], isBackend: Boolean = true) = {
776    val (r_valid, r_idx, r_offset, r_taken, r_mispred) = extractRedirectInfo(redirect)
777    val cfiIndex_bits_wen = r_valid && r_taken && r_offset < cfiIndex_vec(r_idx).bits
778    val cfiIndex_valid_wen = r_valid && r_offset === cfiIndex_vec(r_idx).bits
779    when (cfiIndex_bits_wen || cfiIndex_valid_wen) {
780      cfiIndex_vec(r_idx).valid := cfiIndex_bits_wen || cfiIndex_valid_wen && r_taken
781    }
782    when (cfiIndex_bits_wen) {
783      cfiIndex_vec(r_idx).bits := r_offset
784    }
785    update_target(r_idx) := redirect.bits.cfiUpdate.target
786    if (isBackend) {
787      mispredict_vec(r_idx)(r_offset) := r_mispred
788    }
789  }
790
791  when(backendRedirectReg.valid && lastIsMispredict) {
792    updateCfiInfo(backendRedirectReg)
793  }.elsewhen (ifuRedirectToBpu.valid) {
794    updateCfiInfo(ifuRedirectToBpu, isBackend=false)
795  }
796
797  // ***********************************************************************************
798  // **************************** flush ptr and state queue ****************************
799  // ***********************************************************************************
800
801  val redirectVec = VecInit(backendRedirect, fromIfuRedirect)
802
803  // when redirect, we should reset ptrs and status queues
804  when(redirectVec.map(r => r.valid).reduce(_||_)){
805    val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits)))
806    val notIfu = redirectVec.dropRight(1).map(r => r.valid).reduce(_||_)
807    val (idx, offset, flushItSelf) = (r.ftqIdx, r.ftqOffset, RedirectLevel.flushItself(r.level))
808    val next = idx + 1.U
809    bpuPtr := next
810    ifuPtr := next
811    ifuWbPtr := next
812    when (notIfu) {
813      commitStateQueue(idx.value).zipWithIndex.foreach({ case (s, i) =>
814        when(i.U > offset || i.U === offset && flushItSelf){
815          s := c_invalid
816        }
817      })
818    }
819  }
820
821  // only the valid bit is actually needed
822  io.toIfu.redirect.bits    := backendRedirect.bits
823  io.toIfu.redirect.valid   := stage2Flush
824
825  // commit
826  for (c <- io.fromBackend.rob_commits) {
827    when(c.valid) {
828      commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset) := c_commited
829      // TODO: remove this
830      // For instruction fusions, we also update the next instruction
831      when (c.bits.commitType === 4.U) {
832        commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 1.U) := c_commited
833      }.elsewhen(c.bits.commitType === 5.U) {
834        commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 2.U) := c_commited
835      }.elsewhen(c.bits.commitType === 6.U) {
836        val index = (c.bits.ftqIdx + 1.U).value
837        commitStateQueue(index)(0) := c_commited
838      }.elsewhen(c.bits.commitType === 7.U) {
839        val index = (c.bits.ftqIdx + 1.U).value
840        commitStateQueue(index)(1) := c_commited
841      }
842    }
843  }
844
845  // ****************************************************************
846  // **************************** to bpu ****************************
847  // ****************************************************************
848
849  io.toBpu.redirect <> Mux(fromBackendRedirect.valid, fromBackendRedirect, ifuRedirectToBpu)
850
851  val may_have_stall_from_bpu = RegInit(false.B)
852  val canCommit = commPtr =/= ifuWbPtr && !may_have_stall_from_bpu &&
853    Cat(commitStateQueue(commPtr.value).map(s => {
854      s === c_invalid || s === c_commited
855    })).andR()
856
857  // commit reads
858  ftq_pc_mem.io.raddr.last := commPtr.value
859  val commit_pc_bundle = ftq_pc_mem.io.rdata.last
860  ftq_pd_mem.io.raddr.last := commPtr.value
861  val commit_pd = ftq_pd_mem.io.rdata.last
862  ftq_redirect_sram.io.ren.last := canCommit
863  ftq_redirect_sram.io.raddr.last := commPtr.value
864  val commit_spec_meta = ftq_redirect_sram.io.rdata.last
865  ftq_meta_1r_sram.io.ren(0) := canCommit
866  ftq_meta_1r_sram.io.raddr(0) := commPtr.value
867  val commit_meta = ftq_meta_1r_sram.io.rdata(0)
868  ftb_entry_mem.io.raddr.last := commPtr.value
869  val commit_ftb_entry = ftb_entry_mem.io.rdata.last
870
871  // need one cycle to read mem and srams
872  val do_commit_ptr = RegNext(commPtr)
873  val do_commit = RegNext(canCommit, init=false.B)
874  when (canCommit) { commPtr := commPtr + 1.U }
875  val commit_state = RegNext(commitStateQueue(commPtr.value))
876  val can_commit_cfi = WireInit(cfiIndex_vec(commPtr.value))
877  when (commitStateQueue(commPtr.value)(can_commit_cfi.bits) =/= c_commited) {
878    can_commit_cfi.valid := false.B
879  }
880  val commit_cfi = RegNext(can_commit_cfi)
881
882  val commit_mispredict = VecInit((RegNext(mispredict_vec(commPtr.value)) zip commit_state).map {
883    case (mis, state) => mis && state === c_commited
884  })
885  val can_commit_hit = entry_hit_status(commPtr.value)
886  val commit_hit = RegNext(can_commit_hit)
887  val commit_target = RegNext(update_target(commPtr.value))
888  val commit_stage = RegNext(pred_stage(commPtr.value))
889  val commit_valid = commit_hit === h_hit || commit_cfi.valid // hit or taken
890
891  val to_bpu_hit = can_commit_hit === h_hit || can_commit_hit === h_false_hit
892  may_have_stall_from_bpu := can_commit_cfi.valid && !to_bpu_hit && !may_have_stall_from_bpu
893
894  io.toBpu.update := DontCare
895  io.toBpu.update.valid := commit_valid && do_commit
896  val update = io.toBpu.update.bits
897  update.false_hit   := commit_hit === h_false_hit
898  update.pc          := commit_pc_bundle.startAddr
899  update.meta        := commit_meta.meta
900  update.full_target := commit_target
901  update.from_stage  := commit_stage
902  update.fromFtqRedirectSram(commit_spec_meta)
903
904  val commit_real_hit = commit_hit === h_hit
905  val update_ftb_entry = update.ftb_entry
906
907  val ftbEntryGen = Module(new FTBEntryGen).io
908  ftbEntryGen.start_addr     := commit_pc_bundle.startAddr
909  ftbEntryGen.old_entry      := commit_ftb_entry
910  ftbEntryGen.pd             := commit_pd
911  ftbEntryGen.cfiIndex       := commit_cfi
912  ftbEntryGen.target         := commit_target
913  ftbEntryGen.hit            := commit_real_hit
914  ftbEntryGen.mispredict_vec := commit_mispredict
915
916  update_ftb_entry         := ftbEntryGen.new_entry
917  update.new_br_insert_pos := ftbEntryGen.new_br_insert_pos
918  update.mispred_mask      := ftbEntryGen.mispred_mask
919  update.old_entry         := ftbEntryGen.is_old_entry
920  update.pred_hit          := commit_hit === h_hit || commit_hit === h_false_hit
921
922  update.is_minimal := false.B
923  update.full_pred.fromFtbEntry(ftbEntryGen.new_entry, update.pc)
924  update.full_pred.br_taken_mask  := ftbEntryGen.taken_mask
925  update.full_pred.jalr_target := commit_target
926  update.full_pred.hit := true.B
927  when (update.full_pred.is_jalr) {
928    update.full_pred.targets.last := commit_target
929  }
930
931  // ****************************************************************
932  // *********************** to prefetch ****************************
933  // ****************************************************************
934
935  if(cacheParams.hasPrefetch){
936    val prefetchPtr = RegInit(FtqPtr(false.B, 0.U))
937    prefetchPtr := prefetchPtr + io.toPrefetch.req.fire()
938
939    when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect && !isBefore(prefetchPtr, bpu_s2_resp.ftq_idx)) {
940      prefetchPtr := bpu_s2_resp.ftq_idx
941    }
942
943    when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect && !isBefore(prefetchPtr, bpu_s3_resp.ftq_idx)) {
944      prefetchPtr := bpu_s3_resp.ftq_idx
945      // XSError(true.B, "\ns3_redirect mechanism not implemented!\n")
946    }
947
948    io.toPrefetch.req.valid := prefetchPtr =/= bpuPtr && entry_fetch_status(prefetchPtr.value) === f_to_send
949    io.toPrefetch.req.bits.target := update_target(prefetchPtr.value)
950
951    when(redirectVec.map(r => r.valid).reduce(_||_)){
952      val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits)))
953      val next = r.ftqIdx + 1.U
954      prefetchPtr := next
955    }
956
957    XSError(isBefore(bpuPtr, prefetchPtr) && !isFull(bpuPtr, prefetchPtr), "\nprefetchPtr is before bpuPtr!\n")
958  }
959  else {
960    io.toPrefetch.req <> DontCare
961  }
962
963  // ******************************************************************************
964  // **************************** commit perf counters ****************************
965  // ******************************************************************************
966
967  val commit_inst_mask    = VecInit(commit_state.map(c => c === c_commited && do_commit)).asUInt
968  val commit_mispred_mask = commit_mispredict.asUInt
969  val commit_not_mispred_mask = ~commit_mispred_mask
970
971  val commit_br_mask = commit_pd.brMask.asUInt
972  val commit_jmp_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.jmpInfo.valid.asTypeOf(UInt(1.W)))
973  val commit_cfi_mask = (commit_br_mask | commit_jmp_mask)
974
975  val mbpInstrs = commit_inst_mask & commit_cfi_mask
976
977  val mbpRights = mbpInstrs & commit_not_mispred_mask
978  val mbpWrongs = mbpInstrs & commit_mispred_mask
979
980  io.bpuInfo.bpRight := PopCount(mbpRights)
981  io.bpuInfo.bpWrong := PopCount(mbpWrongs)
982
983  // Cfi Info
984  for (i <- 0 until PredictWidth) {
985    val pc = commit_pc_bundle.startAddr + (i * instBytes).U
986    val v = commit_state(i) === c_commited
987    val isBr = commit_pd.brMask(i)
988    val isJmp = commit_pd.jmpInfo.valid && commit_pd.jmpOffset === i.U
989    val isCfi = isBr || isJmp
990    val isTaken = commit_cfi.valid && commit_cfi.bits === i.U
991    val misPred = commit_mispredict(i)
992    // val ghist = commit_spec_meta.ghist.predHist
993    val histPtr = commit_spec_meta.histPtr
994    val predCycle = commit_meta.meta(63, 0)
995    val target = commit_target
996
997    val brIdx = OHToUInt(Reverse(Cat(update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U})))
998    val inFtbEntry = update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U}.reduce(_||_)
999    val addIntoHist = ((commit_hit === h_hit) && inFtbEntry) || ((!(commit_hit === h_hit) && i.U === commit_cfi.bits && isBr && commit_cfi.valid))
1000    XSDebug(v && do_commit && isCfi, p"cfi_update: isBr(${isBr}) pc(${Hexadecimal(pc)}) " +
1001    p"taken(${isTaken}) mispred(${misPred}) cycle($predCycle) hist(${histPtr.value}) " +
1002    p"startAddr(${Hexadecimal(commit_pc_bundle.startAddr)}) AddIntoHist(${addIntoHist}) " +
1003    p"brInEntry(${inFtbEntry}) brIdx(${brIdx}) target(${Hexadecimal(target)})\n")
1004  }
1005
1006  val enq = io.fromBpu.resp
1007  val perf_redirect = io.fromBackend.redirect
1008
1009  XSPerfAccumulate("entry", validEntries)
1010  XSPerfAccumulate("bpu_to_ftq_stall", enq.valid && !enq.ready)
1011  XSPerfAccumulate("mispredictRedirect", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level)
1012  XSPerfAccumulate("replayRedirect", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level))
1013  XSPerfAccumulate("predecodeRedirect", fromIfuRedirect.valid)
1014
1015  XSPerfAccumulate("to_ifu_bubble", io.toIfu.req.ready && !io.toIfu.req.valid)
1016
1017  XSPerfAccumulate("to_ifu_stall", io.toIfu.req.valid && !io.toIfu.req.ready)
1018  XSPerfAccumulate("from_bpu_real_bubble", !enq.valid && enq.ready && allowBpuIn)
1019  XSPerfAccumulate("bpu_to_ifu_bubble", bpuPtr === ifuPtr)
1020
1021  val from_bpu = io.fromBpu.resp.bits
1022  def in_entry_len_map_gen(resp: BranchPredictionBundle)(stage: String) = {
1023    assert(!resp.is_minimal)
1024    val entry_len = (resp.ftb_entry.getFallThrough(resp.pc) - resp.pc) >> instOffsetBits
1025    val entry_len_recording_vec = (1 to PredictWidth+1).map(i => entry_len === i.U)
1026    val entry_len_map = (1 to PredictWidth+1).map(i =>
1027      f"${stage}_ftb_entry_len_$i" -> (entry_len_recording_vec(i-1) && resp.valid)
1028    ).foldLeft(Map[String, UInt]())(_+_)
1029    entry_len_map
1030  }
1031  val s2_entry_len_map = in_entry_len_map_gen(from_bpu.s2)("s2")
1032  val s3_entry_len_map = in_entry_len_map_gen(from_bpu.s3)("s3")
1033
1034  val to_ifu = io.toIfu.req.bits
1035
1036
1037
1038  val commit_num_inst_recording_vec = (1 to PredictWidth).map(i => PopCount(commit_inst_mask) === i.U)
1039  val commit_num_inst_map = (1 to PredictWidth).map(i =>
1040    f"commit_num_inst_$i" -> (commit_num_inst_recording_vec(i-1) && do_commit)
1041  ).foldLeft(Map[String, UInt]())(_+_)
1042
1043
1044
1045  val commit_jal_mask  = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJal.asTypeOf(UInt(1.W)))
1046  val commit_jalr_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJalr.asTypeOf(UInt(1.W)))
1047  val commit_call_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasCall.asTypeOf(UInt(1.W)))
1048  val commit_ret_mask  = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasRet.asTypeOf(UInt(1.W)))
1049
1050
1051  val mbpBRights = mbpRights & commit_br_mask
1052  val mbpJRights = mbpRights & commit_jal_mask
1053  val mbpIRights = mbpRights & commit_jalr_mask
1054  val mbpCRights = mbpRights & commit_call_mask
1055  val mbpRRights = mbpRights & commit_ret_mask
1056
1057  val mbpBWrongs = mbpWrongs & commit_br_mask
1058  val mbpJWrongs = mbpWrongs & commit_jal_mask
1059  val mbpIWrongs = mbpWrongs & commit_jalr_mask
1060  val mbpCWrongs = mbpWrongs & commit_call_mask
1061  val mbpRWrongs = mbpWrongs & commit_ret_mask
1062
1063  val commit_pred_stage = RegNext(pred_stage(commPtr.value))
1064
1065  def pred_stage_map(src: UInt, name: String) = {
1066    (0 until numBpStages).map(i =>
1067      f"${name}_stage_${i+1}" -> PopCount(src.asBools.map(_ && commit_pred_stage === BP_STAGES(i)))
1068    ).foldLeft(Map[String, UInt]())(_+_)
1069  }
1070
1071  val mispred_stage_map      = pred_stage_map(mbpWrongs,  "mispredict")
1072  val br_mispred_stage_map   = pred_stage_map(mbpBWrongs, "br_mispredict")
1073  val jalr_mispred_stage_map = pred_stage_map(mbpIWrongs, "jalr_mispredict")
1074  val correct_stage_map      = pred_stage_map(mbpRights,  "correct")
1075  val br_correct_stage_map   = pred_stage_map(mbpBRights, "br_correct")
1076  val jalr_correct_stage_map = pred_stage_map(mbpIRights, "jalr_correct")
1077
1078  val update_valid = io.toBpu.update.valid
1079  def u(cond: Bool) = update_valid && cond
1080  val ftb_false_hit = u(update.false_hit)
1081  // assert(!ftb_false_hit)
1082  val ftb_hit = u(commit_hit === h_hit)
1083
1084  val ftb_new_entry = u(ftbEntryGen.is_init_entry)
1085  val ftb_new_entry_only_br = ftb_new_entry && !update_ftb_entry.jmpValid
1086  val ftb_new_entry_only_jmp = ftb_new_entry && !update_ftb_entry.brValids(0)
1087  val ftb_new_entry_has_br_and_jmp = ftb_new_entry && update_ftb_entry.brValids(0) && update_ftb_entry.jmpValid
1088
1089  val ftb_old_entry = u(ftbEntryGen.is_old_entry)
1090
1091  val ftb_modified_entry = u(ftbEntryGen.is_new_br || ftbEntryGen.is_jalr_target_modified || ftbEntryGen.is_always_taken_modified)
1092  val ftb_modified_entry_new_br = u(ftbEntryGen.is_new_br)
1093  val ftb_modified_entry_jalr_target_modified = u(ftbEntryGen.is_jalr_target_modified)
1094  val ftb_modified_entry_br_full = ftb_modified_entry && ftbEntryGen.is_br_full
1095  val ftb_modified_entry_always_taken = ftb_modified_entry && ftbEntryGen.is_always_taken_modified
1096
1097  val ftb_entry_len = (ftbEntryGen.new_entry.getFallThrough(update.pc) - update.pc) >> instOffsetBits
1098  val ftb_entry_len_recording_vec = (1 to PredictWidth+1).map(i => ftb_entry_len === i.U)
1099  val ftb_init_entry_len_map = (1 to PredictWidth+1).map(i =>
1100    f"ftb_init_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_new_entry)
1101  ).foldLeft(Map[String, UInt]())(_+_)
1102  val ftb_modified_entry_len_map = (1 to PredictWidth+1).map(i =>
1103    f"ftb_modified_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_modified_entry)
1104  ).foldLeft(Map[String, UInt]())(_+_)
1105
1106  val ftq_occupancy_map = (0 to FtqSize).map(i =>
1107    f"ftq_has_entry_$i" ->( validEntries === i.U)
1108  ).foldLeft(Map[String, UInt]())(_+_)
1109
1110  val perfCountsMap = Map(
1111    "BpInstr" -> PopCount(mbpInstrs),
1112    "BpBInstr" -> PopCount(mbpBRights | mbpBWrongs),
1113    "BpRight"  -> PopCount(mbpRights),
1114    "BpWrong"  -> PopCount(mbpWrongs),
1115    "BpBRight" -> PopCount(mbpBRights),
1116    "BpBWrong" -> PopCount(mbpBWrongs),
1117    "BpJRight" -> PopCount(mbpJRights),
1118    "BpJWrong" -> PopCount(mbpJWrongs),
1119    "BpIRight" -> PopCount(mbpIRights),
1120    "BpIWrong" -> PopCount(mbpIWrongs),
1121    "BpCRight" -> PopCount(mbpCRights),
1122    "BpCWrong" -> PopCount(mbpCWrongs),
1123    "BpRRight" -> PopCount(mbpRRights),
1124    "BpRWrong" -> PopCount(mbpRWrongs),
1125
1126    "ftb_false_hit"                -> PopCount(ftb_false_hit),
1127    "ftb_hit"                      -> PopCount(ftb_hit),
1128    "ftb_new_entry"                -> PopCount(ftb_new_entry),
1129    "ftb_new_entry_only_br"        -> PopCount(ftb_new_entry_only_br),
1130    "ftb_new_entry_only_jmp"       -> PopCount(ftb_new_entry_only_jmp),
1131    "ftb_new_entry_has_br_and_jmp" -> PopCount(ftb_new_entry_has_br_and_jmp),
1132    "ftb_old_entry"                -> PopCount(ftb_old_entry),
1133    "ftb_modified_entry"           -> PopCount(ftb_modified_entry),
1134    "ftb_modified_entry_new_br"    -> PopCount(ftb_modified_entry_new_br),
1135    "ftb_jalr_target_modified"     -> PopCount(ftb_modified_entry_jalr_target_modified),
1136    "ftb_modified_entry_br_full"   -> PopCount(ftb_modified_entry_br_full),
1137    "ftb_modified_entry_always_taken" -> PopCount(ftb_modified_entry_always_taken)
1138  ) ++ ftb_init_entry_len_map ++ ftb_modified_entry_len_map ++ s2_entry_len_map ++
1139  s3_entry_len_map ++ commit_num_inst_map ++ ftq_occupancy_map ++
1140  mispred_stage_map ++ br_mispred_stage_map ++ jalr_mispred_stage_map ++
1141  correct_stage_map ++ br_correct_stage_map ++ jalr_correct_stage_map
1142
1143  for((key, value) <- perfCountsMap) {
1144    XSPerfAccumulate(key, value)
1145  }
1146
1147  // --------------------------- Debug --------------------------------
1148  // XSDebug(enq_fire, p"enq! " + io.fromBpu.resp.bits.toPrintable)
1149  XSDebug(io.toIfu.req.fire, p"fire to ifu " + io.toIfu.req.bits.toPrintable)
1150  XSDebug(do_commit, p"deq! [ptr] $do_commit_ptr\n")
1151  XSDebug(true.B, p"[bpuPtr] $bpuPtr, [ifuPtr] $ifuPtr, [ifuWbPtr] $ifuWbPtr [commPtr] $commPtr\n")
1152  XSDebug(true.B, p"[in] v:${io.fromBpu.resp.valid} r:${io.fromBpu.resp.ready} " +
1153    p"[out] v:${io.toIfu.req.valid} r:${io.toIfu.req.ready}\n")
1154  XSDebug(do_commit, p"[deq info] cfiIndex: $commit_cfi, $commit_pc_bundle, target: ${Hexadecimal(commit_target)}\n")
1155
1156  //   def ubtbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1157  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1158  //       case (((valid, pd), ans), taken) =>
1159  //       Mux(valid && pd.isBr,
1160  //         isWrong ^ Mux(ans.hit.asBool,
1161  //           Mux(ans.taken.asBool, taken && ans.target === commitEntry.target,
1162  //           !taken),
1163  //         !taken),
1164  //       false.B)
1165  //     }
1166  //   }
1167
1168  //   def btbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1169  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1170  //       case (((valid, pd), ans), taken) =>
1171  //       Mux(valid && pd.isBr,
1172  //         isWrong ^ Mux(ans.hit.asBool,
1173  //           Mux(ans.taken.asBool, taken && ans.target === commitEntry.target,
1174  //           !taken),
1175  //         !taken),
1176  //       false.B)
1177  //     }
1178  //   }
1179
1180  //   def tageCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1181  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1182  //       case (((valid, pd), ans), taken) =>
1183  //       Mux(valid && pd.isBr,
1184  //         isWrong ^ (ans.taken.asBool === taken),
1185  //       false.B)
1186  //     }
1187  //   }
1188
1189  //   def loopCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1190  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1191  //       case (((valid, pd), ans), taken) =>
1192  //       Mux(valid && (pd.isBr) && ans.hit.asBool,
1193  //         isWrong ^ (!taken),
1194  //           false.B)
1195  //     }
1196  //   }
1197
1198  //   def rasCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1199  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1200  //       case (((valid, pd), ans), taken) =>
1201  //       Mux(valid && pd.isRet.asBool /*&& taken*/ && ans.hit.asBool,
1202  //         isWrong ^ (ans.target === commitEntry.target),
1203  //           false.B)
1204  //     }
1205  //   }
1206
1207  //   val ubtbRights = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), false.B)
1208  //   val ubtbWrongs = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), true.B)
1209  //   // btb and ubtb pred jal and jalr as well
1210  //   val btbRights = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), false.B)
1211  //   val btbWrongs = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), true.B)
1212  //   val tageRights = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), false.B)
1213  //   val tageWrongs = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), true.B)
1214
1215  //   val loopRights = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), false.B)
1216  //   val loopWrongs = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), true.B)
1217
1218  //   val rasRights = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), false.B)
1219  //   val rasWrongs = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), true.B)
1220
1221  val perfEvents = Seq(
1222    ("bpu_s2_redirect        ", bpu_s2_redirect                                                             ),
1223    ("bpu_s3_redirect        ", bpu_s3_redirect                                                             ),
1224    ("bpu_to_ftq_stall       ", enq.valid && ~enq.ready                                                     ),
1225    ("mispredictRedirect     ", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level),
1226    ("replayRedirect         ", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level)  ),
1227    ("predecodeRedirect      ", fromIfuRedirect.valid                                                       ),
1228    ("to_ifu_bubble          ", io.toIfu.req.ready && !io.toIfu.req.valid                                   ),
1229    ("from_bpu_real_bubble   ", !enq.valid && enq.ready && allowBpuIn                                       ),
1230    ("BpInstr                ", PopCount(mbpInstrs)                                                         ),
1231    ("BpBInstr               ", PopCount(mbpBRights | mbpBWrongs)                                           ),
1232    ("BpRight                ", PopCount(mbpRights)                                                         ),
1233    ("BpWrong                ", PopCount(mbpWrongs)                                                         ),
1234    ("BpBRight               ", PopCount(mbpBRights)                                                        ),
1235    ("BpBWrong               ", PopCount(mbpBWrongs)                                                        ),
1236    ("BpJRight               ", PopCount(mbpJRights)                                                        ),
1237    ("BpJWrong               ", PopCount(mbpJWrongs)                                                        ),
1238    ("BpIRight               ", PopCount(mbpIRights)                                                        ),
1239    ("BpIWrong               ", PopCount(mbpIWrongs)                                                        ),
1240    ("BpCRight               ", PopCount(mbpCRights)                                                        ),
1241    ("BpCWrong               ", PopCount(mbpCWrongs)                                                        ),
1242    ("BpRRight               ", PopCount(mbpRRights)                                                        ),
1243    ("BpRWrong               ", PopCount(mbpRWrongs)                                                        ),
1244    ("ftb_false_hit          ", PopCount(ftb_false_hit)                                                     ),
1245    ("ftb_hit                ", PopCount(ftb_hit)                                                           ),
1246  )
1247  generatePerfEvent()
1248}
1249