xref: /XiangShan/src/main/scala/xiangshan/frontend/NewFtq.scala (revision b37e4b45da2333608f12413931aecdaef46443e4)
1/***************************************************************************************
2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
3* Copyright (c) 2020-2021 Peng Cheng Laboratory
4*
5* XiangShan is licensed under Mulan PSL v2.
6* You can use this software according to the terms and conditions of the Mulan PSL v2.
7* You may obtain a copy of Mulan PSL v2 at:
8*          http://license.coscl.org.cn/MulanPSL2
9*
10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
13*
14* See the Mulan PSL v2 for more details.
15***************************************************************************************/
16
17package xiangshan.frontend
18
19import chipsalliance.rocketchip.config.Parameters
20import chisel3._
21import chisel3.util._
22import utils._
23import xiangshan._
24import xiangshan.backend.CtrlToFtqIO
25
26class FtqPtr(implicit p: Parameters) extends CircularQueuePtr[FtqPtr](
27  p => p(XSCoreParamsKey).FtqSize
28){
29  override def cloneType = (new FtqPtr).asInstanceOf[this.type]
30}
31
32object FtqPtr {
33  def apply(f: Bool, v: UInt)(implicit p: Parameters): FtqPtr = {
34    val ptr = Wire(new FtqPtr)
35    ptr.flag := f
36    ptr.value := v
37    ptr
38  }
39  def inverse(ptr: FtqPtr)(implicit p: Parameters): FtqPtr = {
40    apply(!ptr.flag, ptr.value)
41  }
42}
43
44class FtqNRSRAM[T <: Data](gen: T, numRead: Int)(implicit p: Parameters) extends XSModule {
45
46  val io = IO(new Bundle() {
47    val raddr = Input(Vec(numRead, UInt(log2Up(FtqSize).W)))
48    val ren = Input(Vec(numRead, Bool()))
49    val rdata = Output(Vec(numRead, gen))
50    val waddr = Input(UInt(log2Up(FtqSize).W))
51    val wen = Input(Bool())
52    val wdata = Input(gen)
53  })
54
55  for(i <- 0 until numRead){
56    val sram = Module(new SRAMTemplate(gen, FtqSize))
57    sram.io.r.req.valid := io.ren(i)
58    sram.io.r.req.bits.setIdx := io.raddr(i)
59    io.rdata(i) := sram.io.r.resp.data(0)
60    sram.io.w.req.valid := io.wen
61    sram.io.w.req.bits.setIdx := io.waddr
62    sram.io.w.req.bits.data := VecInit(io.wdata)
63  }
64
65}
66
67class Ftq_RF_Components(implicit p: Parameters) extends XSBundle with BPUUtils {
68  // TODO: move pftAddr, oversize, carry to another mem
69  val startAddr = UInt(VAddrBits.W)
70  val nextLineAddr = UInt(VAddrBits.W)
71  val isNextMask = Vec(PredictWidth, Bool())
72  val oversize = Bool()
73  val fallThruError = Bool()
74  // val carry = Bool()
75  def getPc(offset: UInt) = {
76    def getHigher(pc: UInt) = pc(VAddrBits-1, log2Ceil(PredictWidth)+instOffsetBits+1)
77    def getOffset(pc: UInt) = pc(log2Ceil(PredictWidth)+instOffsetBits, instOffsetBits)
78    Cat(getHigher(Mux(isNextMask(offset) && startAddr(log2Ceil(PredictWidth)+instOffsetBits), nextLineAddr, startAddr)),
79        getOffset(startAddr)+offset, 0.U(instOffsetBits.W))
80  }
81  def fromBranchPrediction(resp: BranchPredictionBundle) = {
82    def carryPos(addr: UInt) = addr(instOffsetBits+log2Ceil(PredictWidth)+1)
83    this.startAddr := resp.pc
84    this.nextLineAddr := resp.pc + (FetchWidth * 4 * 2).U
85    this.isNextMask := VecInit((0 until PredictWidth).map(i =>
86      (resp.pc(log2Ceil(PredictWidth), 1) +& i.U)(log2Ceil(PredictWidth)).asBool()
87    ))
88    this.oversize := resp.oversize
89    this.fallThruError := resp.fallThruError
90    this
91  }
92  override def toPrintable: Printable = {
93    p"startAddr:${Hexadecimal(startAddr)}"
94  }
95}
96
97class Ftq_pd_Entry(implicit p: Parameters) extends XSBundle {
98  val brMask = Vec(PredictWidth, Bool())
99  val jmpInfo = ValidUndirectioned(Vec(3, Bool()))
100  val jmpOffset = UInt(log2Ceil(PredictWidth).W)
101  val jalTarget = UInt(VAddrBits.W)
102  val rvcMask = Vec(PredictWidth, Bool())
103  def hasJal  = jmpInfo.valid && !jmpInfo.bits(0)
104  def hasJalr = jmpInfo.valid && jmpInfo.bits(0)
105  def hasCall = jmpInfo.valid && jmpInfo.bits(1)
106  def hasRet  = jmpInfo.valid && jmpInfo.bits(2)
107
108  def fromPdWb(pdWb: PredecodeWritebackBundle) = {
109    val pds = pdWb.pd
110    this.brMask := VecInit(pds.map(pd => pd.isBr && pd.valid))
111    this.jmpInfo.valid := VecInit(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid)).asUInt.orR
112    this.jmpInfo.bits := ParallelPriorityMux(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid),
113                                             pds.map(pd => VecInit(pd.isJalr, pd.isCall, pd.isRet)))
114    this.jmpOffset := ParallelPriorityEncoder(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid))
115    this.rvcMask := VecInit(pds.map(pd => pd.isRVC))
116    this.jalTarget := pdWb.jalTarget
117  }
118
119  def toPd(offset: UInt) = {
120    require(offset.getWidth == log2Ceil(PredictWidth))
121    val pd = Wire(new PreDecodeInfo)
122    pd.valid := true.B
123    pd.isRVC := rvcMask(offset)
124    val isBr = brMask(offset)
125    val isJalr = offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(0)
126    pd.brType := Cat(offset === jmpOffset && jmpInfo.valid, isJalr || isBr)
127    pd.isCall := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(1)
128    pd.isRet  := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(2)
129    pd
130  }
131}
132
133
134
135class Ftq_Redirect_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst {
136  val rasSp = UInt(log2Ceil(RasSize).W)
137  val rasEntry = new RASEntry
138  // val specCnt = Vec(numBr, UInt(10.W))
139  // val ghist = new ShiftingGlobalHistory
140  val ghr = UInt(UbtbGHRLength.W)
141  val folded_hist = new AllFoldedHistories(foldedGHistInfos)
142  val histPtr = new CGHPtr
143
144  def fromBranchPrediction(resp: BranchPredictionBundle) = {
145    assert(!resp.is_minimal)
146    this.rasSp := resp.rasSp
147    this.rasEntry := resp.rasTop
148    this.ghr := resp.ghr
149    this.folded_hist := resp.folded_hist
150    this.histPtr := resp.histPtr
151    this
152  }
153}
154
155class Ftq_1R_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst {
156  val meta = UInt(MaxMetaLength.W)
157}
158
159class Ftq_Pred_Info(implicit p: Parameters) extends XSBundle {
160  val target = UInt(VAddrBits.W)
161  val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))
162}
163
164// class FtqEntry(implicit p: Parameters) extends XSBundle with HasBPUConst {
165//   val startAddr = UInt(VAddrBits.W)
166//   val fallThruAddr = UInt(VAddrBits.W)
167//   val isNextMask = Vec(PredictWidth, Bool())
168
169//   val meta = UInt(MaxMetaLength.W)
170
171//   val rasSp = UInt(log2Ceil(RasSize).W)
172//   val rasEntry = new RASEntry
173//   val hist = new ShiftingGlobalHistory
174//   val specCnt = Vec(numBr, UInt(10.W))
175
176//   val valids = Vec(PredictWidth, Bool())
177//   val brMask = Vec(PredictWidth, Bool())
178//   // isJalr, isCall, isRet
179//   val jmpInfo = ValidUndirectioned(Vec(3, Bool()))
180//   val jmpOffset = UInt(log2Ceil(PredictWidth).W)
181
182//   val mispredVec = Vec(PredictWidth, Bool())
183//   val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))
184//   val target = UInt(VAddrBits.W)
185// }
186
187class FtqRead[T <: Data](private val gen: T)(implicit p: Parameters) extends XSBundle {
188  val ptr = Output(new FtqPtr)
189  val offset = Output(UInt(log2Ceil(PredictWidth).W))
190  val data = Input(gen)
191  def apply(ptr: FtqPtr, offset: UInt) = {
192    this.ptr := ptr
193    this.offset := offset
194    this.data
195  }
196  override def cloneType = (new FtqRead(gen)).asInstanceOf[this.type]
197}
198
199
200class FtqToBpuIO(implicit p: Parameters) extends XSBundle {
201  val redirect = Valid(new BranchPredictionRedirect)
202  val update = Valid(new BranchPredictionUpdate)
203  val enq_ptr = Output(new FtqPtr)
204}
205
206class FtqToIfuIO(implicit p: Parameters) extends XSBundle with HasCircularQueuePtrHelper {
207  val req = Decoupled(new FetchRequestBundle)
208  val redirect = Valid(new Redirect)
209  val flushFromBpu = new Bundle {
210    // when ifu pipeline is not stalled,
211    // a packet from bpu s3 can reach f1 at most
212    val s2 = Valid(new FtqPtr)
213    // val s3 = Valid(new FtqPtr)
214    def shouldFlushBy(src: Valid[FtqPtr], idx_to_flush: FtqPtr) = {
215      src.valid && !isAfter(src.bits, idx_to_flush)
216    }
217    def shouldFlushByStage2(idx: FtqPtr) = shouldFlushBy(s2, idx)
218    // def shouldFlushByStage3(idx: FtqPtr) = shouldFlushBy(s3, idx)
219  }
220}
221
222trait HasBackendRedirectInfo extends HasXSParameter {
223  def numRedirect = exuParameters.JmpCnt + exuParameters.AluCnt + 1
224  def isLoadReplay(r: Valid[Redirect]) = r.bits.flushItself()
225}
226
227class FtqToCtrlIO(implicit p: Parameters) extends XSBundle with HasBackendRedirectInfo {
228  val pc_reads = Vec(1 + numRedirect + 1 + 1, Flipped(new FtqRead(UInt(VAddrBits.W))))
229  val target_read = Flipped(new FtqRead(UInt(VAddrBits.W)))
230  def getJumpPcRead = pc_reads.head
231  def getRedirectPcRead = VecInit(pc_reads.tail.dropRight(2))
232  def getMemPredPcRead = pc_reads.init.last
233  def getRobFlushPcRead = pc_reads.last
234}
235
236
237class FTBEntryGen(implicit p: Parameters) extends XSModule with HasBackendRedirectInfo with HasBPUParameter {
238  val io = IO(new Bundle {
239    val start_addr = Input(UInt(VAddrBits.W))
240    val old_entry = Input(new FTBEntry)
241    val pd = Input(new Ftq_pd_Entry)
242    val cfiIndex = Flipped(Valid(UInt(log2Ceil(PredictWidth).W)))
243    val target = Input(UInt(VAddrBits.W))
244    val hit = Input(Bool())
245    val mispredict_vec = Input(Vec(PredictWidth, Bool()))
246
247    val new_entry = Output(new FTBEntry)
248    val new_br_insert_pos = Output(Vec(numBr, Bool()))
249    val taken_mask = Output(Vec(numBr, Bool()))
250    val mispred_mask = Output(Vec(numBr+1, Bool()))
251
252    // for perf counters
253    val is_init_entry = Output(Bool())
254    val is_old_entry = Output(Bool())
255    val is_new_br = Output(Bool())
256    val is_jalr_target_modified = Output(Bool())
257    val is_always_taken_modified = Output(Bool())
258    val is_br_full = Output(Bool())
259  })
260
261  // no mispredictions detected at predecode
262  val hit = io.hit
263  val pd = io.pd
264
265  val init_entry = WireInit(0.U.asTypeOf(new FTBEntry))
266
267
268  val cfi_is_br = pd.brMask(io.cfiIndex.bits) && io.cfiIndex.valid
269  val entry_has_jmp = pd.jmpInfo.valid
270  val new_jmp_is_jal  = entry_has_jmp && !pd.jmpInfo.bits(0) && io.cfiIndex.valid
271  val new_jmp_is_jalr = entry_has_jmp &&  pd.jmpInfo.bits(0) && io.cfiIndex.valid
272  val new_jmp_is_call = entry_has_jmp &&  pd.jmpInfo.bits(1) && io.cfiIndex.valid
273  val new_jmp_is_ret  = entry_has_jmp &&  pd.jmpInfo.bits(2) && io.cfiIndex.valid
274  val last_jmp_rvi = entry_has_jmp && pd.jmpOffset === (PredictWidth-1).U && !pd.rvcMask.last
275  val last_br_rvi = cfi_is_br && io.cfiIndex.bits === (PredictWidth-1).U && !pd.rvcMask.last
276
277  val cfi_is_jal = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jal
278  val cfi_is_jalr = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jalr
279
280  def carryPos = log2Ceil(PredictWidth)+instOffsetBits+1
281  def getLower(pc: UInt) = pc(carryPos-1, instOffsetBits)
282  // if not hit, establish a new entry
283  init_entry.valid := true.B
284  // tag is left for ftb to assign
285
286  // case br
287  val init_br_slot = init_entry.getSlotForBr(0)
288  when (cfi_is_br) {
289    init_br_slot.valid := true.B
290    init_br_slot.offset := io.cfiIndex.bits
291    init_br_slot.setLowerStatByTarget(io.start_addr, io.target, numBr == 1)
292    init_entry.always_taken(0) := true.B // set to always taken on init
293  }
294
295  // case jmp
296  when (entry_has_jmp) {
297    init_entry.tailSlot.offset := pd.jmpOffset
298    init_entry.tailSlot.valid := new_jmp_is_jal || new_jmp_is_jalr
299    init_entry.tailSlot.setLowerStatByTarget(io.start_addr, Mux(cfi_is_jalr, io.target, pd.jalTarget), isShare=false)
300  }
301
302  val jmpPft = getLower(io.start_addr) +& pd.jmpOffset +& Mux(pd.rvcMask(pd.jmpOffset), 1.U, 2.U)
303  init_entry.pftAddr := Mux(entry_has_jmp, jmpPft, getLower(io.start_addr) + ((FetchWidth*4)>>instOffsetBits).U + Mux(last_br_rvi, 1.U, 0.U))
304  init_entry.carry   := Mux(entry_has_jmp, jmpPft(carryPos-instOffsetBits), io.start_addr(carryPos-1) || (io.start_addr(carryPos-2, instOffsetBits).andR && last_br_rvi))
305  init_entry.isJalr := new_jmp_is_jalr
306  init_entry.isCall := new_jmp_is_call
307  init_entry.isRet  := new_jmp_is_ret
308  init_entry.last_is_rvc := Mux(entry_has_jmp, pd.rvcMask(pd.jmpOffset), pd.rvcMask.last)
309
310  init_entry.oversize := last_br_rvi || last_jmp_rvi
311
312  // if hit, check whether a new cfi(only br is possible) is detected
313  val oe = io.old_entry
314  val br_recorded_vec = oe.getBrRecordedVec(io.cfiIndex.bits)
315  val br_recorded = br_recorded_vec.asUInt.orR
316  val is_new_br = cfi_is_br && !br_recorded
317  val new_br_offset = io.cfiIndex.bits
318  // vec(i) means new br will be inserted BEFORE old br(i)
319  val allBrSlotsVec = oe.allSlotsForBr
320  val new_br_insert_onehot = VecInit((0 until numBr).map{
321    i => i match {
322      case 0 =>
323        !allBrSlotsVec(0).valid || new_br_offset < allBrSlotsVec(0).offset
324      case idx =>
325        allBrSlotsVec(idx-1).valid && new_br_offset > allBrSlotsVec(idx-1).offset &&
326        (!allBrSlotsVec(idx).valid || new_br_offset < allBrSlotsVec(idx).offset)
327    }
328  })
329
330  val old_entry_modified = WireInit(io.old_entry)
331  for (i <- 0 until numBr) {
332    val slot = old_entry_modified.allSlotsForBr(i)
333    when (new_br_insert_onehot(i)) {
334      slot.valid := true.B
335      slot.offset := new_br_offset
336      slot.setLowerStatByTarget(io.start_addr, io.target, i == numBr-1)
337      old_entry_modified.always_taken(i) := true.B
338    }.elsewhen (new_br_offset > oe.allSlotsForBr(i).offset) {
339      old_entry_modified.always_taken(i) := false.B
340      // all other fields remain unchanged
341    }.otherwise {
342      // case i == 0, remain unchanged
343      if (i != 0) {
344        val noNeedToMoveFromFormerSlot = (i == numBr-1).B && !oe.brSlots.last.valid
345        when (!noNeedToMoveFromFormerSlot) {
346          slot.fromAnotherSlot(oe.allSlotsForBr(i-1))
347          old_entry_modified.always_taken(i) := oe.always_taken(i)
348        }
349      }
350    }
351  }
352
353  // two circumstances:
354  // 1. oe: | br | j  |, new br should be in front of j, thus addr of j should be new pft
355  // 2. oe: | br | br |, new br could be anywhere between, thus new pft is the addr of either
356  //        the previous last br or the new br
357  val may_have_to_replace = oe.noEmptySlotForNewBr
358  val pft_need_to_change = is_new_br && may_have_to_replace
359  // it should either be the given last br or the new br
360  when (pft_need_to_change) {
361    val new_pft_offset =
362      Mux(!new_br_insert_onehot.asUInt.orR,
363        new_br_offset, oe.allSlotsForBr.last.offset)
364
365    // set jmp to invalid
366    old_entry_modified.pftAddr := getLower(io.start_addr) + new_pft_offset
367    old_entry_modified.last_is_rvc := pd.rvcMask(new_pft_offset - 1.U) // TODO: fix this
368    old_entry_modified.carry := (getLower(io.start_addr) +& new_pft_offset).head(1).asBool
369    old_entry_modified.oversize := false.B
370    old_entry_modified.isCall := false.B
371    old_entry_modified.isRet := false.B
372    old_entry_modified.isJalr := false.B
373  }
374
375  val old_entry_jmp_target_modified = WireInit(oe)
376  val old_target = oe.tailSlot.getTarget(io.start_addr) // may be wrong because we store only 20 lowest bits
377  val old_tail_is_jmp = !oe.tailSlot.sharing
378  val jalr_target_modified = cfi_is_jalr && (old_target =/= io.target) && old_tail_is_jmp // TODO: pass full jalr target
379  when (jalr_target_modified) {
380    old_entry_jmp_target_modified.setByJmpTarget(io.start_addr, io.target)
381    old_entry_jmp_target_modified.always_taken := 0.U.asTypeOf(Vec(numBr, Bool()))
382  }
383
384  val old_entry_always_taken = WireInit(oe)
385  val always_taken_modified_vec = Wire(Vec(numBr, Bool())) // whether modified or not
386  for (i <- 0 until numBr) {
387    old_entry_always_taken.always_taken(i) :=
388      oe.always_taken(i) && io.cfiIndex.valid && oe.brValids(i) && io.cfiIndex.bits === oe.brOffset(i)
389    always_taken_modified_vec(i) := oe.always_taken(i) && !old_entry_always_taken.always_taken(i)
390  }
391  val always_taken_modified = always_taken_modified_vec.reduce(_||_)
392
393
394
395  val derived_from_old_entry =
396    Mux(is_new_br, old_entry_modified,
397      Mux(jalr_target_modified, old_entry_jmp_target_modified, old_entry_always_taken))
398
399
400  io.new_entry := Mux(!hit, init_entry, derived_from_old_entry)
401
402  io.new_br_insert_pos := new_br_insert_onehot
403  io.taken_mask := VecInit((io.new_entry.brOffset zip io.new_entry.brValids).map{
404    case (off, v) => io.cfiIndex.bits === off && io.cfiIndex.valid && v
405  })
406  for (i <- 0 until numBr) {
407    io.mispred_mask(i) := io.new_entry.brValids(i) && io.mispredict_vec(io.new_entry.brOffset(i))
408  }
409  io.mispred_mask.last := io.new_entry.jmpValid && io.mispredict_vec(pd.jmpOffset)
410
411  // for perf counters
412  io.is_init_entry := !hit
413  io.is_old_entry := hit && !is_new_br && !jalr_target_modified && !always_taken_modified
414  io.is_new_br := hit && is_new_br
415  io.is_jalr_target_modified := hit && jalr_target_modified
416  io.is_always_taken_modified := hit && always_taken_modified
417  io.is_br_full := hit && is_new_br && may_have_to_replace
418}
419
420class Ftq(implicit p: Parameters) extends XSModule with HasCircularQueuePtrHelper
421  with HasBackendRedirectInfo with BPUUtils with HasBPUConst with HasPerfEvents {
422  val io = IO(new Bundle {
423    val fromBpu = Flipped(new BpuToFtqIO)
424    val fromIfu = Flipped(new IfuToFtqIO)
425    val fromBackend = Flipped(new CtrlToFtqIO)
426
427    val toBpu = new FtqToBpuIO
428    val toIfu = new FtqToIfuIO
429    val toBackend = new FtqToCtrlIO
430
431    val bpuInfo = new Bundle {
432      val bpRight = Output(UInt(XLEN.W))
433      val bpWrong = Output(UInt(XLEN.W))
434    }
435  })
436  io.bpuInfo := DontCare
437
438  val stage2Redirect = io.fromBackend.stage2Redirect
439  val stage3Redirect = RegNext(io.fromBackend.stage2Redirect)
440
441  val stage2Flush = stage2Redirect.valid
442  val backendFlush = stage2Flush || RegNext(stage2Flush)
443  val ifuFlush = Wire(Bool())
444
445  val flush = stage2Flush || RegNext(stage2Flush)
446
447  val allowBpuIn, allowToIfu = WireInit(false.B)
448  val flushToIfu = !allowToIfu
449  allowBpuIn := !ifuFlush && !stage2Redirect.valid && !stage3Redirect.valid
450  allowToIfu := !ifuFlush && !stage2Redirect.valid && !stage3Redirect.valid
451
452  val bpuPtr, ifuPtr, ifuWbPtr, commPtr = RegInit(FtqPtr(false.B, 0.U))
453  val validEntries = distanceBetween(bpuPtr, commPtr)
454
455  // **********************************************************************
456  // **************************** enq from bpu ****************************
457  // **********************************************************************
458  val new_entry_ready = validEntries < FtqSize.U
459  io.fromBpu.resp.ready := new_entry_ready
460
461  val bpu_s2_resp = io.fromBpu.resp.bits.s2
462  // val bpu_s3_resp = io.fromBpu.resp.bits.s3
463  val bpu_s2_redirect = bpu_s2_resp.valid && bpu_s2_resp.hasRedirect
464  // val bpu_s3_redirect = bpu_s3_resp.valid && bpu_s3_resp.hasRedirect
465
466  io.toBpu.enq_ptr := bpuPtr
467  val enq_fire = io.fromBpu.resp.fire() && allowBpuIn // from bpu s1
468  val bpu_in_fire = (io.fromBpu.resp.fire() || bpu_s2_redirect/*  || bpu_s3_redirect */) && allowBpuIn
469
470  val bpu_in_resp = io.fromBpu.resp.bits.selectedResp
471  val bpu_in_stage = io.fromBpu.resp.bits.selectedRespIdx
472  val bpu_in_resp_ptr = Mux(bpu_in_stage === BP_S1, bpuPtr, bpu_in_resp.ftq_idx)
473  val bpu_in_resp_idx = bpu_in_resp_ptr.value
474
475  // read ports:                            jumpPc + redirects + loadPred + robFlush + ifuReq1 + ifuReq2 + commitUpdate
476  val ftq_pc_mem = Module(new SyncDataModuleTemplate(new Ftq_RF_Components, FtqSize, 1+numRedirect+2+1+1+1, 1))
477  // resp from uBTB
478  ftq_pc_mem.io.wen(0) := bpu_in_fire
479  ftq_pc_mem.io.waddr(0) := bpu_in_resp_idx
480  ftq_pc_mem.io.wdata(0).fromBranchPrediction(bpu_in_resp)
481
482  //                                                            ifuRedirect + backendRedirect + commit
483  val ftq_redirect_sram = Module(new FtqNRSRAM(new Ftq_Redirect_SRAMEntry, 1+1+1))
484  // these info is intended to enq at the last stage of bpu
485  ftq_redirect_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid
486  ftq_redirect_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value
487  ftq_redirect_sram.io.wdata.fromBranchPrediction(io.fromBpu.resp.bits.lastStage)
488
489  val ftq_meta_1r_sram = Module(new FtqNRSRAM(new Ftq_1R_SRAMEntry, 1))
490  // these info is intended to enq at the last stage of bpu
491  ftq_meta_1r_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid
492  ftq_meta_1r_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value
493  ftq_meta_1r_sram.io.wdata.meta := io.fromBpu.resp.bits.meta
494  //                                                            ifuRedirect + backendRedirect + commit
495  val ftb_entry_mem = Module(new SyncDataModuleTemplate(new FTBEntry, FtqSize, 1+1+1, 1))
496  ftb_entry_mem.io.wen(0) := io.fromBpu.resp.bits.lastStage.valid
497  ftb_entry_mem.io.waddr(0) := io.fromBpu.resp.bits.lastStage.ftq_idx.value
498  ftb_entry_mem.io.wdata(0) := io.fromBpu.resp.bits.lastStage.ftb_entry
499
500
501  // multi-write
502  val update_target = Reg(Vec(FtqSize, UInt(VAddrBits.W))) // could be taken target or fallThrough
503  val cfiIndex_vec = Reg(Vec(FtqSize, ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))))
504  val mispredict_vec = Reg(Vec(FtqSize, Vec(PredictWidth, Bool())))
505  val pred_stage = Reg(Vec(FtqSize, UInt(2.W)))
506
507  val c_invalid :: c_valid :: c_commited :: Nil = Enum(3)
508  val commitStateQueue = RegInit(VecInit(Seq.fill(FtqSize) {
509    VecInit(Seq.fill(PredictWidth)(c_invalid))
510  }))
511
512  val f_to_send :: f_sent :: Nil = Enum(2)
513  val entry_fetch_status = RegInit(VecInit(Seq.fill(FtqSize)(f_sent)))
514
515  val h_not_hit :: h_false_hit :: h_hit :: Nil = Enum(3)
516  val entry_hit_status = RegInit(VecInit(Seq.fill(FtqSize)(h_not_hit)))
517
518
519  when (bpu_in_fire) {
520    entry_fetch_status(bpu_in_resp_idx) := f_to_send
521    commitStateQueue(bpu_in_resp_idx) := VecInit(Seq.fill(PredictWidth)(c_invalid))
522    cfiIndex_vec(bpu_in_resp_idx) := bpu_in_resp.cfiIndex
523    mispredict_vec(bpu_in_resp_idx) := WireInit(VecInit(Seq.fill(PredictWidth)(false.B)))
524    update_target(bpu_in_resp_idx) := bpu_in_resp.getTarget
525    pred_stage(bpu_in_resp_idx) := bpu_in_stage
526  }
527
528  bpuPtr := bpuPtr + enq_fire
529  ifuPtr := ifuPtr + io.toIfu.req.fire
530
531  // only use ftb result to assign hit status
532  when (bpu_s2_resp.valid) {
533    entry_hit_status(bpu_s2_resp.ftq_idx.value) := Mux(bpu_s2_resp.full_pred.hit, h_hit, h_not_hit)
534  }
535
536
537  io.toIfu.flushFromBpu.s2.valid := bpu_s2_redirect
538  io.toIfu.flushFromBpu.s2.bits := bpu_s2_resp.ftq_idx
539  when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect) {
540    bpuPtr := bpu_s2_resp.ftq_idx + 1.U
541    // only when ifuPtr runs ahead of bpu s2 resp should we recover it
542    when (!isBefore(ifuPtr, bpu_s2_resp.ftq_idx)) {
543      ifuPtr := bpu_s2_resp.ftq_idx
544    }
545  }
546
547  // io.toIfu.flushFromBpu.s3.valid := bpu_s3_redirect
548  // io.toIfu.flushFromBpu.s3.bits := bpu_s3_resp.ftq_idx
549  // when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect) {
550  //   bpuPtr := bpu_s3_resp.ftq_idx + 1.U
551  //   // only when ifuPtr runs ahead of bpu s2 resp should we recover it
552  //   when (!isBefore(ifuPtr, bpu_s3_resp.ftq_idx)) {
553  //     ifuPtr := bpu_s3_resp.ftq_idx
554  //   }
555  //   XSError(true.B, "\ns3_redirect mechanism not implemented!\n")
556  // }
557
558  XSError(isBefore(bpuPtr, ifuPtr) && !isFull(bpuPtr, ifuPtr), "\nifuPtr is before bpuPtr!\n")
559
560  // ****************************************************************
561  // **************************** to ifu ****************************
562  // ****************************************************************
563  val bpu_in_bypass_buf = RegEnable(ftq_pc_mem.io.wdata(0), enable=bpu_in_fire)
564  val bpu_in_bypass_ptr = RegNext(bpu_in_resp_ptr)
565  val last_cycle_bpu_in = RegNext(bpu_in_fire)
566  val last_cycle_to_ifu_fire = RegNext(io.toIfu.req.fire)
567
568  // read pc and target
569  ftq_pc_mem.io.raddr.init.init.last := ifuPtr.value
570  ftq_pc_mem.io.raddr.init.last := (ifuPtr+1.U).value
571
572  io.toIfu.req.valid := allowToIfu && entry_fetch_status(ifuPtr.value) === f_to_send && ifuPtr =/= bpuPtr
573  io.toIfu.req.bits.ftqIdx := ifuPtr
574  io.toIfu.req.bits.nextStartAddr := update_target(ifuPtr.value)
575  io.toIfu.req.bits.ftqOffset := cfiIndex_vec(ifuPtr.value)
576
577  val toIfuPcBundle = Wire(new Ftq_RF_Components)
578  when (last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr) {
579    toIfuPcBundle := bpu_in_bypass_buf
580  }.elsewhen (last_cycle_to_ifu_fire) {
581    toIfuPcBundle := ftq_pc_mem.io.rdata.init.last
582  }.otherwise {
583    toIfuPcBundle := ftq_pc_mem.io.rdata.init.init.last
584  }
585
586  io.toIfu.req.bits.fromFtqPcBundle(toIfuPcBundle)
587
588  // when fall through is smaller in value than start address, there must be a false hit
589  when (toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit) {
590    when (io.toIfu.req.fire &&
591      !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr)/*  &&
592      !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr) */
593    ) {
594      entry_hit_status(ifuPtr.value) := h_false_hit
595      XSError(true.B, "FTB false hit by fallThroughError, startAddr: %x, fallTHru: %x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr)
596    }
597    XSDebug(true.B, "fallThruError! start:%x, fallThru:%x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr)
598  }
599
600  val ifu_req_should_be_flushed =
601    io.toIfu.flushFromBpu.shouldFlushByStage2(io.toIfu.req.bits.ftqIdx)/*  ||
602    io.toIfu.flushFromBpu.shouldFlushByStage3(io.toIfu.req.bits.ftqIdx) */
603
604  when (io.toIfu.req.fire && !ifu_req_should_be_flushed) {
605    entry_fetch_status(ifuPtr.value) := f_sent
606  }
607
608
609  // *********************************************************************
610  // **************************** wb from ifu ****************************
611  // *********************************************************************
612  val pdWb = io.fromIfu.pdWb
613  val pds = pdWb.bits.pd
614  val ifu_wb_valid = pdWb.valid
615  val ifu_wb_idx = pdWb.bits.ftqIdx.value
616  // read ports:                                                         commit update
617  val ftq_pd_mem = Module(new SyncDataModuleTemplate(new Ftq_pd_Entry, FtqSize, 1, 1))
618  ftq_pd_mem.io.wen(0) := ifu_wb_valid
619  ftq_pd_mem.io.waddr(0) := pdWb.bits.ftqIdx.value
620  ftq_pd_mem.io.wdata(0).fromPdWb(pdWb.bits)
621
622  val hit_pd_valid = entry_hit_status(ifu_wb_idx) === h_hit && ifu_wb_valid
623  val hit_pd_mispred = hit_pd_valid && pdWb.bits.misOffset.valid
624  val hit_pd_mispred_reg = RegNext(hit_pd_mispred, init=false.B)
625  val pd_reg       = RegEnable(pds,             enable = pdWb.valid)
626  val start_pc_reg = RegEnable(pdWb.bits.pc(0), enable = pdWb.valid)
627  val wb_idx_reg   = RegEnable(ifu_wb_idx,      enable = pdWb.valid)
628
629  when (ifu_wb_valid) {
630    val comm_stq_wen = VecInit(pds.map(_.valid).zip(pdWb.bits.instrRange).map{
631      case (v, inRange) => v && inRange
632    })
633    (commitStateQueue(ifu_wb_idx) zip comm_stq_wen).map{
634      case (qe, v) => when (v) { qe := c_valid }
635    }
636  }
637
638  ifuWbPtr := ifuWbPtr + ifu_wb_valid
639
640  ftb_entry_mem.io.raddr.head := ifu_wb_idx
641  val has_false_hit = WireInit(false.B)
642  when (RegNext(hit_pd_valid)) {
643    // check for false hit
644    val pred_ftb_entry = ftb_entry_mem.io.rdata.head
645    val brSlots = pred_ftb_entry.brSlots
646    val tailSlot = pred_ftb_entry.tailSlot
647    // we check cfis that bpu predicted
648
649    // bpu predicted branches but denied by predecode
650    val br_false_hit =
651      brSlots.map{
652        s => s.valid && !(pd_reg(s.offset).valid && pd_reg(s.offset).isBr)
653      }.reduce(_||_) ||
654      (tailSlot.valid && pred_ftb_entry.tailSlot.sharing &&
655        !(pd_reg(tailSlot.offset).valid && pd_reg(tailSlot.offset).isBr))
656
657    val jmpOffset = tailSlot.offset
658    val jmp_pd = pd_reg(jmpOffset)
659    val jal_false_hit = pred_ftb_entry.jmpValid &&
660      ((pred_ftb_entry.isJal  && !(jmp_pd.valid && jmp_pd.isJal)) ||
661       (pred_ftb_entry.isJalr && !(jmp_pd.valid && jmp_pd.isJalr)) ||
662       (pred_ftb_entry.isCall && !(jmp_pd.valid && jmp_pd.isCall)) ||
663       (pred_ftb_entry.isRet  && !(jmp_pd.valid && jmp_pd.isRet))
664      )
665
666    has_false_hit := br_false_hit || jal_false_hit || hit_pd_mispred_reg
667    XSDebug(has_false_hit, "FTB false hit by br or jal or hit_pd, startAddr: %x\n", pdWb.bits.pc(0))
668
669    assert(!has_false_hit)
670  }
671
672  when (has_false_hit) {
673    entry_hit_status(wb_idx_reg) := h_false_hit
674  }
675
676
677  // **********************************************************************
678  // **************************** backend read ****************************
679  // **********************************************************************
680
681  // pc reads
682  for ((req, i) <- io.toBackend.pc_reads.zipWithIndex) {
683    ftq_pc_mem.io.raddr(i) := req.ptr.value
684    req.data := ftq_pc_mem.io.rdata(i).getPc(RegNext(req.offset))
685  }
686  // target read
687  io.toBackend.target_read.data := RegNext(update_target(io.toBackend.target_read.ptr.value))
688
689  // *******************************************************************************
690  // **************************** redirect from backend ****************************
691  // *******************************************************************************
692
693  // redirect read cfiInfo, couples to redirectGen s2
694  ftq_redirect_sram.io.ren.init.last := io.fromBackend.stage2Redirect.valid
695  ftq_redirect_sram.io.raddr.init.last := io.fromBackend.stage2Redirect.bits.ftqIdx.value
696
697  ftb_entry_mem.io.raddr.init.last := io.fromBackend.stage2Redirect.bits.ftqIdx.value
698
699  val stage3CfiInfo = ftq_redirect_sram.io.rdata.init.last
700  val fromBackendRedirect = WireInit(stage3Redirect)
701  val backendRedirectCfi = fromBackendRedirect.bits.cfiUpdate
702  backendRedirectCfi.fromFtqRedirectSram(stage3CfiInfo)
703
704  val r_ftb_entry = ftb_entry_mem.io.rdata.init.last
705  val r_ftqOffset = fromBackendRedirect.bits.ftqOffset
706
707  when (entry_hit_status(fromBackendRedirect.bits.ftqIdx.value) === h_hit) {
708    backendRedirectCfi.shift := PopCount(r_ftb_entry.getBrMaskByOffset(r_ftqOffset)) +&
709      (backendRedirectCfi.pd.isBr && !r_ftb_entry.brIsSaved(r_ftqOffset) &&
710      !r_ftb_entry.newBrCanNotInsert(r_ftqOffset))
711
712    backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr && (r_ftb_entry.brIsSaved(r_ftqOffset) ||
713        !r_ftb_entry.newBrCanNotInsert(r_ftqOffset))
714  }.otherwise {
715    backendRedirectCfi.shift := (backendRedirectCfi.pd.isBr && backendRedirectCfi.taken).asUInt
716    backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr.asUInt
717  }
718
719
720  // ***************************************************************************
721  // **************************** redirect from ifu ****************************
722  // ***************************************************************************
723  val fromIfuRedirect = WireInit(0.U.asTypeOf(Valid(new Redirect)))
724  fromIfuRedirect.valid := pdWb.valid && pdWb.bits.misOffset.valid && !backendFlush
725  fromIfuRedirect.bits.ftqIdx := pdWb.bits.ftqIdx
726  fromIfuRedirect.bits.ftqOffset := pdWb.bits.misOffset.bits
727  fromIfuRedirect.bits.level := RedirectLevel.flushAfter
728
729  val ifuRedirectCfiUpdate = fromIfuRedirect.bits.cfiUpdate
730  ifuRedirectCfiUpdate.pc := pdWb.bits.pc(pdWb.bits.misOffset.bits)
731  ifuRedirectCfiUpdate.pd := pdWb.bits.pd(pdWb.bits.misOffset.bits)
732  ifuRedirectCfiUpdate.predTaken := cfiIndex_vec(pdWb.bits.ftqIdx.value).valid
733  ifuRedirectCfiUpdate.target := pdWb.bits.target
734  ifuRedirectCfiUpdate.taken := pdWb.bits.cfiOffset.valid
735  ifuRedirectCfiUpdate.isMisPred := pdWb.bits.misOffset.valid
736
737  val ifuRedirectReg = RegNext(fromIfuRedirect, init=0.U.asTypeOf(Valid(new Redirect)))
738  val ifuRedirectToBpu = WireInit(ifuRedirectReg)
739  ifuFlush := fromIfuRedirect.valid || ifuRedirectToBpu.valid
740
741  ftq_redirect_sram.io.ren.head := fromIfuRedirect.valid
742  ftq_redirect_sram.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value
743
744  ftb_entry_mem.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value
745
746  val toBpuCfi = ifuRedirectToBpu.bits.cfiUpdate
747  toBpuCfi.fromFtqRedirectSram(ftq_redirect_sram.io.rdata.head)
748  when (ifuRedirectReg.bits.cfiUpdate.pd.isRet) {
749    toBpuCfi.target := toBpuCfi.rasEntry.retAddr
750  }
751
752  // *********************************************************************
753  // **************************** wb from exu ****************************
754  // *********************************************************************
755
756  def extractRedirectInfo(wb: Valid[Redirect]) = {
757    val ftqIdx = wb.bits.ftqIdx.value
758    val ftqOffset = wb.bits.ftqOffset
759    val taken = wb.bits.cfiUpdate.taken
760    val mispred = wb.bits.cfiUpdate.isMisPred
761    (wb.valid, ftqIdx, ftqOffset, taken, mispred)
762  }
763
764  // fix mispredict entry
765  val lastIsMispredict = RegNext(
766    stage2Redirect.valid && stage2Redirect.bits.level === RedirectLevel.flushAfter, init = false.B
767  )
768
769  def updateCfiInfo(redirect: Valid[Redirect], isBackend: Boolean = true) = {
770    val (r_valid, r_idx, r_offset, r_taken, r_mispred) = extractRedirectInfo(redirect)
771    val cfiIndex_bits_wen = r_valid && r_taken && r_offset < cfiIndex_vec(r_idx).bits
772    val cfiIndex_valid_wen = r_valid && r_offset === cfiIndex_vec(r_idx).bits
773    when (cfiIndex_bits_wen || cfiIndex_valid_wen) {
774      cfiIndex_vec(r_idx).valid := cfiIndex_bits_wen || cfiIndex_valid_wen && r_taken
775    }
776    when (cfiIndex_bits_wen) {
777      cfiIndex_vec(r_idx).bits := r_offset
778    }
779    update_target(r_idx) := redirect.bits.cfiUpdate.target
780    if (isBackend) {
781      mispredict_vec(r_idx)(r_offset) := r_mispred
782    }
783  }
784
785  when(stage3Redirect.valid && lastIsMispredict) {
786    updateCfiInfo(stage3Redirect)
787  }.elsewhen (ifuRedirectToBpu.valid) {
788    updateCfiInfo(ifuRedirectToBpu, isBackend=false)
789  }
790
791  // ***********************************************************************************
792  // **************************** flush ptr and state queue ****************************
793  // ***********************************************************************************
794
795  val redirectVec = VecInit(stage2Redirect, fromIfuRedirect)
796
797  // when redirect, we should reset ptrs and status queues
798  when(redirectVec.map(r => r.valid).reduce(_||_)){
799    val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits)))
800    val notIfu = redirectVec.dropRight(1).map(r => r.valid).reduce(_||_)
801    val (idx, offset, flushItSelf) = (r.ftqIdx, r.ftqOffset, RedirectLevel.flushItself(r.level))
802    val next = idx + 1.U
803    bpuPtr := next
804    ifuPtr := next
805    ifuWbPtr := next
806    when (notIfu) {
807      commitStateQueue(idx.value).zipWithIndex.foreach({ case (s, i) =>
808        when(i.U > offset || i.U === offset && flushItSelf){
809          s := c_invalid
810        }
811      })
812    }
813  }
814
815  // only the valid bit is actually needed
816  io.toIfu.redirect.bits    := stage2Redirect.bits
817  io.toIfu.redirect.valid   := stage2Flush
818
819  // commit
820  for (c <- io.fromBackend.rob_commits) {
821    when(c.valid) {
822      commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset) := c_commited
823      // TODO: remove this
824      // For instruction fusions, we also update the next instruction
825      when (c.bits.commitType === 4.U) {
826        commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 1.U) := c_commited
827      }.elsewhen(c.bits.commitType === 5.U) {
828        commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 2.U) := c_commited
829      }.elsewhen(c.bits.commitType === 6.U) {
830        val index = (c.bits.ftqIdx + 1.U).value
831        commitStateQueue(index)(0) := c_commited
832      }.elsewhen(c.bits.commitType === 7.U) {
833        val index = (c.bits.ftqIdx + 1.U).value
834        commitStateQueue(index)(1) := c_commited
835      }
836    }
837  }
838
839  // ****************************************************************
840  // **************************** to bpu ****************************
841  // ****************************************************************
842
843  io.toBpu.redirect <> Mux(fromBackendRedirect.valid, fromBackendRedirect, ifuRedirectToBpu)
844
845  val may_have_stall_from_bpu = RegInit(false.B)
846  val canCommit = commPtr =/= ifuWbPtr && !may_have_stall_from_bpu &&
847    Cat(commitStateQueue(commPtr.value).map(s => {
848      s === c_invalid || s === c_commited
849    })).andR()
850
851  // commit reads
852  ftq_pc_mem.io.raddr.last := commPtr.value
853  val commit_pc_bundle = ftq_pc_mem.io.rdata.last
854  ftq_pd_mem.io.raddr.last := commPtr.value
855  val commit_pd = ftq_pd_mem.io.rdata.last
856  ftq_redirect_sram.io.ren.last := canCommit
857  ftq_redirect_sram.io.raddr.last := commPtr.value
858  val commit_spec_meta = ftq_redirect_sram.io.rdata.last
859  ftq_meta_1r_sram.io.ren(0) := canCommit
860  ftq_meta_1r_sram.io.raddr(0) := commPtr.value
861  val commit_meta = ftq_meta_1r_sram.io.rdata(0)
862  ftb_entry_mem.io.raddr.last := commPtr.value
863  val commit_ftb_entry = ftb_entry_mem.io.rdata.last
864
865  // need one cycle to read mem and srams
866  val do_commit_ptr = RegNext(commPtr)
867  val do_commit = RegNext(canCommit, init=false.B)
868  when (canCommit) { commPtr := commPtr + 1.U }
869  val commit_state = RegNext(commitStateQueue(commPtr.value))
870  val can_commit_cfi = WireInit(cfiIndex_vec(commPtr.value))
871  when (commitStateQueue(commPtr.value)(can_commit_cfi.bits) =/= c_commited) {
872    can_commit_cfi.valid := false.B
873  }
874  val commit_cfi = RegNext(can_commit_cfi)
875
876  val commit_mispredict = VecInit((RegNext(mispredict_vec(commPtr.value)) zip commit_state).map {
877    case (mis, state) => mis && state === c_commited
878  })
879  val can_commit_hit = entry_hit_status(commPtr.value)
880  val commit_hit = RegNext(can_commit_hit)
881  val commit_target = RegNext(update_target(commPtr.value))
882  val commit_valid = commit_hit === h_hit || commit_cfi.valid // hit or taken
883
884  val to_bpu_hit = can_commit_hit === h_hit || can_commit_hit === h_false_hit
885  may_have_stall_from_bpu := can_commit_cfi.valid && !to_bpu_hit && !may_have_stall_from_bpu
886
887  io.toBpu.update := DontCare
888  io.toBpu.update.valid := commit_valid && do_commit
889  val update = io.toBpu.update.bits
890  update.false_hit   := commit_hit === h_false_hit
891  update.pc          := commit_pc_bundle.startAddr
892  update.meta        := commit_meta.meta
893  update.full_target := commit_target
894  update.fromFtqRedirectSram(commit_spec_meta)
895
896  val commit_real_hit = commit_hit === h_hit
897  val update_ftb_entry = update.ftb_entry
898
899  val ftbEntryGen = Module(new FTBEntryGen).io
900  ftbEntryGen.start_addr     := commit_pc_bundle.startAddr
901  ftbEntryGen.old_entry      := commit_ftb_entry
902  ftbEntryGen.pd             := commit_pd
903  ftbEntryGen.cfiIndex       := commit_cfi
904  ftbEntryGen.target         := commit_target
905  ftbEntryGen.hit            := commit_real_hit
906  ftbEntryGen.mispredict_vec := commit_mispredict
907
908  update_ftb_entry         := ftbEntryGen.new_entry
909  update.new_br_insert_pos := ftbEntryGen.new_br_insert_pos
910  update.mispred_mask      := ftbEntryGen.mispred_mask
911  update.old_entry         := ftbEntryGen.is_old_entry
912
913  update.is_minimal := false.B
914  update.full_pred.fromFtbEntry(ftbEntryGen.new_entry, update.pc)
915  update.full_pred.br_taken_mask  := ftbEntryGen.taken_mask
916  update.full_pred.jalr_target := commit_target
917  update.full_pred.hit := true.B
918  when (update.full_pred.is_jalr) {
919    update.full_pred.targets.last := commit_target
920  }
921
922  // ******************************************************************************
923  // **************************** commit perf counters ****************************
924  // ******************************************************************************
925
926  val commit_inst_mask    = VecInit(commit_state.map(c => c === c_commited && do_commit)).asUInt
927  val commit_mispred_mask = commit_mispredict.asUInt
928  val commit_not_mispred_mask = ~commit_mispred_mask
929
930  val commit_br_mask = commit_pd.brMask.asUInt
931  val commit_jmp_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.jmpInfo.valid.asTypeOf(UInt(1.W)))
932  val commit_cfi_mask = (commit_br_mask | commit_jmp_mask)
933
934  val mbpInstrs = commit_inst_mask & commit_cfi_mask
935
936  val mbpRights = mbpInstrs & commit_not_mispred_mask
937  val mbpWrongs = mbpInstrs & commit_mispred_mask
938
939  io.bpuInfo.bpRight := PopCount(mbpRights)
940  io.bpuInfo.bpWrong := PopCount(mbpWrongs)
941
942  // Cfi Info
943  for (i <- 0 until PredictWidth) {
944    val pc = commit_pc_bundle.startAddr + (i * instBytes).U
945    val v = commit_state(i) === c_commited
946    val isBr = commit_pd.brMask(i)
947    val isJmp = commit_pd.jmpInfo.valid && commit_pd.jmpOffset === i.U
948    val isCfi = isBr || isJmp
949    val isTaken = commit_cfi.valid && commit_cfi.bits === i.U
950    val misPred = commit_mispredict(i)
951    // val ghist = commit_spec_meta.ghist.predHist
952    val histPtr = commit_spec_meta.histPtr
953    val predCycle = commit_meta.meta(63, 0)
954    val target = commit_target
955
956    val brIdx = OHToUInt(Reverse(Cat(update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U})))
957    val inFtbEntry = update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U}.reduce(_||_)
958    val addIntoHist = ((commit_hit === h_hit) && inFtbEntry) || ((!(commit_hit === h_hit) && i.U === commit_cfi.bits && isBr && commit_cfi.valid))
959    XSDebug(v && do_commit && isCfi, p"cfi_update: isBr(${isBr}) pc(${Hexadecimal(pc)}) " +
960    p"taken(${isTaken}) mispred(${misPred}) cycle($predCycle) hist(${histPtr.value}) " +
961    p"startAddr(${Hexadecimal(commit_pc_bundle.startAddr)}) AddIntoHist(${addIntoHist}) " +
962    p"brInEntry(${inFtbEntry}) brIdx(${brIdx}) target(${Hexadecimal(target)})\n")
963  }
964
965  val enq = io.fromBpu.resp
966  val perf_redirect = io.fromBackend.stage2Redirect
967
968  XSPerfAccumulate("entry", validEntries)
969  XSPerfAccumulate("bpu_to_ftq_stall", enq.valid && !enq.ready)
970  XSPerfAccumulate("mispredictRedirect", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level)
971  XSPerfAccumulate("replayRedirect", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level))
972  XSPerfAccumulate("predecodeRedirect", fromIfuRedirect.valid)
973
974  XSPerfAccumulate("to_ifu_bubble", io.toIfu.req.ready && !io.toIfu.req.valid)
975
976  XSPerfAccumulate("to_ifu_stall", io.toIfu.req.valid && !io.toIfu.req.ready)
977  XSPerfAccumulate("from_bpu_real_bubble", !enq.valid && enq.ready && allowBpuIn)
978  XSPerfAccumulate("bpu_to_ftq_bubble", bpuPtr === ifuPtr)
979
980  val from_bpu = io.fromBpu.resp.bits
981  def in_entry_len_map_gen(resp: BranchPredictionBundle)(stage: String) = {
982    assert(!resp.is_minimal)
983    val entry_len = (resp.ftb_entry.getFallThrough(resp.pc) - resp.pc) >> instOffsetBits
984    val entry_len_recording_vec = (1 to PredictWidth+1).map(i => entry_len === i.U)
985    val entry_len_map = (1 to PredictWidth+1).map(i =>
986      f"${stage}_ftb_entry_len_$i" -> (entry_len_recording_vec(i-1) && resp.valid)
987    ).foldLeft(Map[String, UInt]())(_+_)
988    entry_len_map
989  }
990  val s2_entry_len_map = in_entry_len_map_gen(from_bpu.s2)("s2")
991
992  val to_ifu = io.toIfu.req.bits
993
994
995
996  val commit_num_inst_recording_vec = (1 to PredictWidth).map(i => PopCount(commit_inst_mask) === i.U)
997  val commit_num_inst_map = (1 to PredictWidth).map(i =>
998    f"commit_num_inst_$i" -> (commit_num_inst_recording_vec(i-1) && do_commit)
999  ).foldLeft(Map[String, UInt]())(_+_)
1000
1001
1002
1003  val commit_jal_mask  = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJal.asTypeOf(UInt(1.W)))
1004  val commit_jalr_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJalr.asTypeOf(UInt(1.W)))
1005  val commit_call_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasCall.asTypeOf(UInt(1.W)))
1006  val commit_ret_mask  = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasRet.asTypeOf(UInt(1.W)))
1007
1008
1009  val mbpBRights = mbpRights & commit_br_mask
1010  val mbpJRights = mbpRights & commit_jal_mask
1011  val mbpIRights = mbpRights & commit_jalr_mask
1012  val mbpCRights = mbpRights & commit_call_mask
1013  val mbpRRights = mbpRights & commit_ret_mask
1014
1015  val mbpBWrongs = mbpWrongs & commit_br_mask
1016  val mbpJWrongs = mbpWrongs & commit_jal_mask
1017  val mbpIWrongs = mbpWrongs & commit_jalr_mask
1018  val mbpCWrongs = mbpWrongs & commit_call_mask
1019  val mbpRWrongs = mbpWrongs & commit_ret_mask
1020
1021  val commit_pred_stage = RegNext(pred_stage(commPtr.value))
1022
1023  def pred_stage_map(src: UInt, name: String) = {
1024    (0 until numBpStages).map(i =>
1025      f"${name}_stage_${i+1}" -> PopCount(src.asBools.map(_ && commit_pred_stage === BP_STAGES(i)))
1026    ).foldLeft(Map[String, UInt]())(_+_)
1027  }
1028
1029  val mispred_stage_map      = pred_stage_map(mbpWrongs,  "mispredict")
1030  val br_mispred_stage_map   = pred_stage_map(mbpBWrongs, "br_mispredict")
1031  val jalr_mispred_stage_map = pred_stage_map(mbpIWrongs, "jalr_mispredict")
1032  val correct_stage_map      = pred_stage_map(mbpRights,  "correct")
1033  val br_correct_stage_map   = pred_stage_map(mbpBRights, "br_correct")
1034  val jalr_correct_stage_map = pred_stage_map(mbpIRights, "jalr_correct")
1035
1036  val update_valid = io.toBpu.update.valid
1037  def u(cond: Bool) = update_valid && cond
1038  val ftb_false_hit = u(update.false_hit)
1039  // assert(!ftb_false_hit)
1040  val ftb_hit = u(commit_hit === h_hit)
1041
1042  val ftb_new_entry = u(ftbEntryGen.is_init_entry)
1043  val ftb_new_entry_only_br = ftb_new_entry && !update_ftb_entry.jmpValid
1044  val ftb_new_entry_only_jmp = ftb_new_entry && !update_ftb_entry.brValids(0)
1045  val ftb_new_entry_has_br_and_jmp = ftb_new_entry && update_ftb_entry.brValids(0) && update_ftb_entry.jmpValid
1046
1047  val ftb_old_entry = u(ftbEntryGen.is_old_entry)
1048
1049  val ftb_modified_entry = u(ftbEntryGen.is_new_br || ftbEntryGen.is_jalr_target_modified || ftbEntryGen.is_always_taken_modified)
1050  val ftb_modified_entry_new_br = u(ftbEntryGen.is_new_br)
1051  val ftb_modified_entry_jalr_target_modified = u(ftbEntryGen.is_jalr_target_modified)
1052  val ftb_modified_entry_br_full = ftb_modified_entry && ftbEntryGen.is_br_full
1053  val ftb_modified_entry_always_taken = ftb_modified_entry && ftbEntryGen.is_always_taken_modified
1054
1055  val ftb_entry_len = (ftbEntryGen.new_entry.getFallThrough(update.pc) - update.pc) >> instOffsetBits
1056  val ftb_entry_len_recording_vec = (1 to PredictWidth+1).map(i => ftb_entry_len === i.U)
1057  val ftb_init_entry_len_map = (1 to PredictWidth+1).map(i =>
1058    f"ftb_init_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_new_entry)
1059  ).foldLeft(Map[String, UInt]())(_+_)
1060  val ftb_modified_entry_len_map = (1 to PredictWidth+1).map(i =>
1061    f"ftb_modified_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_modified_entry)
1062  ).foldLeft(Map[String, UInt]())(_+_)
1063
1064  val ftq_occupancy_map = (0 to FtqSize).map(i =>
1065    f"ftq_has_entry_$i" ->( validEntries === i.U)
1066  ).foldLeft(Map[String, UInt]())(_+_)
1067
1068  val perfCountsMap = Map(
1069    "BpInstr" -> PopCount(mbpInstrs),
1070    "BpBInstr" -> PopCount(mbpBRights | mbpBWrongs),
1071    "BpRight"  -> PopCount(mbpRights),
1072    "BpWrong"  -> PopCount(mbpWrongs),
1073    "BpBRight" -> PopCount(mbpBRights),
1074    "BpBWrong" -> PopCount(mbpBWrongs),
1075    "BpJRight" -> PopCount(mbpJRights),
1076    "BpJWrong" -> PopCount(mbpJWrongs),
1077    "BpIRight" -> PopCount(mbpIRights),
1078    "BpIWrong" -> PopCount(mbpIWrongs),
1079    "BpCRight" -> PopCount(mbpCRights),
1080    "BpCWrong" -> PopCount(mbpCWrongs),
1081    "BpRRight" -> PopCount(mbpRRights),
1082    "BpRWrong" -> PopCount(mbpRWrongs),
1083
1084    "ftb_false_hit"                -> PopCount(ftb_false_hit),
1085    "ftb_hit"                      -> PopCount(ftb_hit),
1086    "ftb_new_entry"                -> PopCount(ftb_new_entry),
1087    "ftb_new_entry_only_br"        -> PopCount(ftb_new_entry_only_br),
1088    "ftb_new_entry_only_jmp"       -> PopCount(ftb_new_entry_only_jmp),
1089    "ftb_new_entry_has_br_and_jmp" -> PopCount(ftb_new_entry_has_br_and_jmp),
1090    "ftb_old_entry"                -> PopCount(ftb_old_entry),
1091    "ftb_modified_entry"           -> PopCount(ftb_modified_entry),
1092    "ftb_modified_entry_new_br"    -> PopCount(ftb_modified_entry_new_br),
1093    "ftb_jalr_target_modified"     -> PopCount(ftb_modified_entry_jalr_target_modified),
1094    "ftb_modified_entry_br_full"   -> PopCount(ftb_modified_entry_br_full),
1095    "ftb_modified_entry_always_taken" -> PopCount(ftb_modified_entry_always_taken)
1096  ) ++ ftb_init_entry_len_map ++ ftb_modified_entry_len_map ++
1097  s2_entry_len_map ++ commit_num_inst_map ++ ftq_occupancy_map ++
1098  mispred_stage_map ++ br_mispred_stage_map ++ jalr_mispred_stage_map ++
1099  correct_stage_map ++ br_correct_stage_map ++ jalr_correct_stage_map
1100
1101  for((key, value) <- perfCountsMap) {
1102    XSPerfAccumulate(key, value)
1103  }
1104
1105  // --------------------------- Debug --------------------------------
1106  // XSDebug(enq_fire, p"enq! " + io.fromBpu.resp.bits.toPrintable)
1107  XSDebug(io.toIfu.req.fire, p"fire to ifu " + io.toIfu.req.bits.toPrintable)
1108  XSDebug(do_commit, p"deq! [ptr] $do_commit_ptr\n")
1109  XSDebug(true.B, p"[bpuPtr] $bpuPtr, [ifuPtr] $ifuPtr, [ifuWbPtr] $ifuWbPtr [commPtr] $commPtr\n")
1110  XSDebug(true.B, p"[in] v:${io.fromBpu.resp.valid} r:${io.fromBpu.resp.ready} " +
1111    p"[out] v:${io.toIfu.req.valid} r:${io.toIfu.req.ready}\n")
1112  XSDebug(do_commit, p"[deq info] cfiIndex: $commit_cfi, $commit_pc_bundle, target: ${Hexadecimal(commit_target)}\n")
1113
1114  //   def ubtbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1115  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1116  //       case (((valid, pd), ans), taken) =>
1117  //       Mux(valid && pd.isBr,
1118  //         isWrong ^ Mux(ans.hit.asBool,
1119  //           Mux(ans.taken.asBool, taken && ans.target === commitEntry.target,
1120  //           !taken),
1121  //         !taken),
1122  //       false.B)
1123  //     }
1124  //   }
1125
1126  //   def btbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1127  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1128  //       case (((valid, pd), ans), taken) =>
1129  //       Mux(valid && pd.isBr,
1130  //         isWrong ^ Mux(ans.hit.asBool,
1131  //           Mux(ans.taken.asBool, taken && ans.target === commitEntry.target,
1132  //           !taken),
1133  //         !taken),
1134  //       false.B)
1135  //     }
1136  //   }
1137
1138  //   def tageCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1139  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1140  //       case (((valid, pd), ans), taken) =>
1141  //       Mux(valid && pd.isBr,
1142  //         isWrong ^ (ans.taken.asBool === taken),
1143  //       false.B)
1144  //     }
1145  //   }
1146
1147  //   def loopCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1148  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1149  //       case (((valid, pd), ans), taken) =>
1150  //       Mux(valid && (pd.isBr) && ans.hit.asBool,
1151  //         isWrong ^ (!taken),
1152  //           false.B)
1153  //     }
1154  //   }
1155
1156  //   def rasCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1157  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1158  //       case (((valid, pd), ans), taken) =>
1159  //       Mux(valid && pd.isRet.asBool /*&& taken*/ && ans.hit.asBool,
1160  //         isWrong ^ (ans.target === commitEntry.target),
1161  //           false.B)
1162  //     }
1163  //   }
1164
1165  //   val ubtbRights = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), false.B)
1166  //   val ubtbWrongs = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), true.B)
1167  //   // btb and ubtb pred jal and jalr as well
1168  //   val btbRights = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), false.B)
1169  //   val btbWrongs = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), true.B)
1170  //   val tageRights = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), false.B)
1171  //   val tageWrongs = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), true.B)
1172
1173  //   val loopRights = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), false.B)
1174  //   val loopWrongs = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), true.B)
1175
1176  //   val rasRights = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), false.B)
1177  //   val rasWrongs = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), true.B)
1178
1179  val perfEvents = Seq(
1180    ("bpu_s2_redirect        ", bpu_s2_redirect                                                             ),
1181    // ("bpu_s3_redirect        ", bpu_s3_redirect                                                             ),
1182    ("bpu_to_ftq_stall       ", enq.valid && ~enq.ready                                                     ),
1183    ("mispredictRedirect     ", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level),
1184    ("replayRedirect         ", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level)  ),
1185    ("predecodeRedirect      ", fromIfuRedirect.valid                                                       ),
1186    ("to_ifu_bubble          ", io.toIfu.req.ready && !io.toIfu.req.valid                                   ),
1187    ("from_bpu_real_bubble   ", !enq.valid && enq.ready && allowBpuIn                                       ),
1188    ("BpInstr                ", PopCount(mbpInstrs)                                                         ),
1189    ("BpBInstr               ", PopCount(mbpBRights | mbpBWrongs)                                           ),
1190    ("BpRight                ", PopCount(mbpRights)                                                         ),
1191    ("BpWrong                ", PopCount(mbpWrongs)                                                         ),
1192    ("BpBRight               ", PopCount(mbpBRights)                                                        ),
1193    ("BpBWrong               ", PopCount(mbpBWrongs)                                                        ),
1194    ("BpJRight               ", PopCount(mbpJRights)                                                        ),
1195    ("BpJWrong               ", PopCount(mbpJWrongs)                                                        ),
1196    ("BpIRight               ", PopCount(mbpIRights)                                                        ),
1197    ("BpIWrong               ", PopCount(mbpIWrongs)                                                        ),
1198    ("BpCRight               ", PopCount(mbpCRights)                                                        ),
1199    ("BpCWrong               ", PopCount(mbpCWrongs)                                                        ),
1200    ("BpRRight               ", PopCount(mbpRRights)                                                        ),
1201    ("BpRWrong               ", PopCount(mbpRWrongs)                                                        ),
1202    ("ftb_false_hit          ", PopCount(ftb_false_hit)                                                     ),
1203    ("ftb_hit                ", PopCount(ftb_hit)                                                           ),
1204  )
1205  generatePerfEvent()
1206}
1207