xref: /XiangShan/src/main/scala/xiangshan/frontend/IBuffer.scala (revision 92c61038092ac384201fa5f56e31f96be14a30f5)
144c9c1deSEaston Man/***************************************************************************************
244c9c1deSEaston Man* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
344c9c1deSEaston Man* Copyright (c) 2020-2021 Peng Cheng Laboratory
444c9c1deSEaston Man*
544c9c1deSEaston Man* XiangShan is licensed under Mulan PSL v2.
644c9c1deSEaston Man* You can use this software according to the terms and conditions of the Mulan PSL v2.
744c9c1deSEaston Man* You may obtain a copy of Mulan PSL v2 at:
844c9c1deSEaston Man*          http://license.coscl.org.cn/MulanPSL2
944c9c1deSEaston Man*
1044c9c1deSEaston Man* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
1144c9c1deSEaston Man* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
1244c9c1deSEaston Man* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
1344c9c1deSEaston Man*
1444c9c1deSEaston Man* See the Mulan PSL v2 for more details.
1544c9c1deSEaston Man***************************************************************************************/
1644c9c1deSEaston Man
1744c9c1deSEaston Manpackage xiangshan.frontend
1844c9c1deSEaston Man
1944c9c1deSEaston Manimport org.chipsalliance.cde.config.Parameters
2044c9c1deSEaston Manimport chisel3._
2144c9c1deSEaston Manimport chisel3.util._
2244c9c1deSEaston Manimport xiangshan._
2344c9c1deSEaston Manimport utils._
2444c9c1deSEaston Manimport utility._
2544c9c1deSEaston Manimport xiangshan.ExceptionNO._
2644c9c1deSEaston Man
2744c9c1deSEaston Manclass IBufPtr(implicit p: Parameters) extends CircularQueuePtr[IBufPtr](
2844c9c1deSEaston Man  p => p(XSCoreParamsKey).IBufSize
2944c9c1deSEaston Man) {
3044c9c1deSEaston Man}
3144c9c1deSEaston Man
3244c9c1deSEaston Manclass IBufInBankPtr(implicit p: Parameters) extends CircularQueuePtr[IBufInBankPtr](
3344c9c1deSEaston Man  p => p(XSCoreParamsKey).IBufSize / p(XSCoreParamsKey).IBufNBank
3444c9c1deSEaston Man) {
3544c9c1deSEaston Man}
3644c9c1deSEaston Man
3744c9c1deSEaston Manclass IBufBankPtr(implicit p: Parameters) extends CircularQueuePtr[IBufBankPtr](
3844c9c1deSEaston Man  p => p(XSCoreParamsKey).IBufNBank
3944c9c1deSEaston Man) {
4044c9c1deSEaston Man}
4144c9c1deSEaston Man
4244c9c1deSEaston Manclass IBufferIO(implicit p: Parameters) extends XSBundle {
4344c9c1deSEaston Man  val flush = Input(Bool())
4444c9c1deSEaston Man  val ControlRedirect = Input(Bool())
4544c9c1deSEaston Man  val ControlBTBMissBubble = Input(Bool())
4644c9c1deSEaston Man  val TAGEMissBubble = Input(Bool())
4744c9c1deSEaston Man  val SCMissBubble = Input(Bool())
4844c9c1deSEaston Man  val ITTAGEMissBubble = Input(Bool())
4944c9c1deSEaston Man  val RASMissBubble = Input(Bool())
5044c9c1deSEaston Man  val MemVioRedirect = Input(Bool())
5144c9c1deSEaston Man  val in = Flipped(DecoupledIO(new FetchToIBuffer))
5244c9c1deSEaston Man  val out = Vec(DecodeWidth, DecoupledIO(new CtrlFlow))
5344c9c1deSEaston Man  val full = Output(Bool())
5405cc2a4eSXuan Hu  val decodeCanAccept = Input(Bool())
5544c9c1deSEaston Man  val stallReason = new StallReasonIO(DecodeWidth)
5644c9c1deSEaston Man}
5744c9c1deSEaston Man
5844c9c1deSEaston Manclass IBufEntry(implicit p: Parameters) extends XSBundle {
5944c9c1deSEaston Man  val inst = UInt(32.W)
6044c9c1deSEaston Man  val pc = UInt(VAddrBits.W)
6144c9c1deSEaston Man  val foldpc = UInt(MemPredPCWidth.W)
6244c9c1deSEaston Man  val pd = new PreDecodeInfo
6344c9c1deSEaston Man  val pred_taken = Bool()
6444c9c1deSEaston Man  val ftqPtr = new FtqPtr
6544c9c1deSEaston Man  val ftqOffset = UInt(log2Ceil(PredictWidth).W)
66*92c61038SXuan Hu  val exceptionType = IBufferExceptionType()
677e0f64b0SGuanghui Cheng  val triggered = TriggerAction()
6844c9c1deSEaston Man
6944c9c1deSEaston Man  def fromFetch(fetch: FetchToIBuffer, i: Int): IBufEntry = {
7044c9c1deSEaston Man    inst   := fetch.instrs(i)
7144c9c1deSEaston Man    pc     := fetch.pc(i)
7244c9c1deSEaston Man    foldpc := fetch.foldpc(i)
7344c9c1deSEaston Man    pd     := fetch.pd(i)
7444c9c1deSEaston Man    pred_taken := fetch.ftqOffset(i).valid
7544c9c1deSEaston Man    ftqPtr := fetch.ftqPtr
7644c9c1deSEaston Man    ftqOffset := fetch.ftqOffset(i).bits
77*92c61038SXuan Hu    exceptionType := IBufferExceptionType.cvtFromFetchExcpAndCrossPageAndRVCII(
78*92c61038SXuan Hu      fetch.exceptionType(i),
79*92c61038SXuan Hu      fetch.crossPageIPFFix(i),
80*92c61038SXuan Hu      fetch.illegalInstr(i),
81*92c61038SXuan Hu    )
8244c9c1deSEaston Man    triggered := fetch.triggered(i)
8344c9c1deSEaston Man    this
8444c9c1deSEaston Man  }
8544c9c1deSEaston Man
8644c9c1deSEaston Man  def toCtrlFlow: CtrlFlow = {
8744c9c1deSEaston Man    val cf = Wire(new CtrlFlow)
8844c9c1deSEaston Man    cf.instr := inst
8944c9c1deSEaston Man    cf.pc := pc
9044c9c1deSEaston Man    cf.foldpc := foldpc
9144c9c1deSEaston Man    cf.exceptionVec := 0.U.asTypeOf(ExceptionVec())
92*92c61038SXuan Hu    cf.exceptionVec(instrPageFault)      := IBufferExceptionType.isPF (this.exceptionType)
93*92c61038SXuan Hu    cf.exceptionVec(instrGuestPageFault) := IBufferExceptionType.isGPF(this.exceptionType)
94*92c61038SXuan Hu    cf.exceptionVec(instrAccessFault)    := IBufferExceptionType.isAF (this.exceptionType)
95*92c61038SXuan Hu    cf.exceptionVec(EX_II)               := IBufferExceptionType.isRVCII(this.exceptionType)
9644c9c1deSEaston Man    cf.trigger := triggered
9744c9c1deSEaston Man    cf.pd := pd
9844c9c1deSEaston Man    cf.pred_taken := pred_taken
99*92c61038SXuan Hu    cf.crossPageIPFFix := IBufferExceptionType.isCrossPage(this.exceptionType)
10044c9c1deSEaston Man    cf.storeSetHit := DontCare
10144c9c1deSEaston Man    cf.waitForRobIdx := DontCare
10244c9c1deSEaston Man    cf.loadWaitBit := DontCare
10344c9c1deSEaston Man    cf.loadWaitStrict := DontCare
10444c9c1deSEaston Man    cf.ssid := DontCare
10544c9c1deSEaston Man    cf.ftqPtr := ftqPtr
10644c9c1deSEaston Man    cf.ftqOffset := ftqOffset
10744c9c1deSEaston Man    cf
10844c9c1deSEaston Man  }
109*92c61038SXuan Hu
110*92c61038SXuan Hu  object IBufferExceptionType extends NamedUInt(3) {
111*92c61038SXuan Hu    def None         = "b000".U
112*92c61038SXuan Hu    def NonCrossPF   = "b001".U
113*92c61038SXuan Hu    def NonCrossGPF  = "b010".U
114*92c61038SXuan Hu    def NonCrossAF   = "b011".U
115*92c61038SXuan Hu    // illegal instruction
116*92c61038SXuan Hu    def rvcII        = "b100".U
117*92c61038SXuan Hu    def CrossPF      = "b101".U
118*92c61038SXuan Hu    def CrossGPF     = "b110".U
119*92c61038SXuan Hu    def CrossAF      = "b111".U
120*92c61038SXuan Hu
121*92c61038SXuan Hu    def cvtFromFetchExcpAndCrossPageAndRVCII(fetchExcp: UInt, crossPage: Bool, rvcIll: Bool): UInt = {
122*92c61038SXuan Hu      require(
123*92c61038SXuan Hu        fetchExcp.getWidth == ExceptionType.width,
124*92c61038SXuan Hu        s"The width(${fetchExcp.getWidth}) of fetchExcp should be equal to " +
125*92c61038SXuan Hu        s"the width(${ExceptionType.width}) of frontend.ExceptionType."
126*92c61038SXuan Hu      )
127*92c61038SXuan Hu      MuxCase(fetchExcp, Seq(
128*92c61038SXuan Hu        rvcIll    -> this.rvcII,
129*92c61038SXuan Hu        crossPage -> Cat(1.U(1.W), fetchExcp),
130*92c61038SXuan Hu      ))
131*92c61038SXuan Hu    }
132*92c61038SXuan Hu
133*92c61038SXuan Hu    def isRVCII(uint: UInt): Bool = {
134*92c61038SXuan Hu      this.checkInputWidth(uint)
135*92c61038SXuan Hu      uint(2) && uint(1, 0) === 0.U
136*92c61038SXuan Hu    }
137*92c61038SXuan Hu
138*92c61038SXuan Hu    def isCrossPage(uint: UInt): Bool = {
139*92c61038SXuan Hu      this.checkInputWidth(uint)
140*92c61038SXuan Hu      uint(2) && uint(1, 0) =/= 0.U
141*92c61038SXuan Hu    }
142*92c61038SXuan Hu
143*92c61038SXuan Hu    def isPF (uint: UInt): Bool = uint(1, 0) === this.NonCrossPF (1, 0)
144*92c61038SXuan Hu    def isGPF(uint: UInt): Bool = uint(1, 0) === this.NonCrossGPF(1, 0)
145*92c61038SXuan Hu    def isAF (uint: UInt): Bool = uint(1, 0) === this.NonCrossAF (1, 0)
146*92c61038SXuan Hu  }
14744c9c1deSEaston Man}
14844c9c1deSEaston Man
14944c9c1deSEaston Manclass IBuffer(implicit p: Parameters) extends XSModule with HasCircularQueuePtrHelper with HasPerfEvents {
15044c9c1deSEaston Man  val io = IO(new IBufferIO)
15144c9c1deSEaston Man
15205cc2a4eSXuan Hu  // io alias
15305cc2a4eSXuan Hu  private val decodeCanAccept = io.decodeCanAccept
15405cc2a4eSXuan Hu
15544c9c1deSEaston Man  // Parameter Check
15644c9c1deSEaston Man  private val bankSize = IBufSize / IBufNBank
15744c9c1deSEaston Man  require(IBufSize % IBufNBank == 0, s"IBufNBank should divide IBufSize, IBufNBank: $IBufNBank, IBufSize: $IBufSize")
15844c9c1deSEaston Man  require(IBufNBank >= DecodeWidth,
15944c9c1deSEaston Man    s"IBufNBank should be equal or larger than DecodeWidth, IBufNBank: $IBufNBank, DecodeWidth: $DecodeWidth")
16044c9c1deSEaston Man
16144c9c1deSEaston Man  // IBuffer is organized as raw registers
16244c9c1deSEaston Man  // This is due to IBuffer is a huge queue, read & write port logic should be precisely controlled
16344c9c1deSEaston Man  //                             . + + E E E - .
16444c9c1deSEaston Man  //                             . + + E E E - .
16544c9c1deSEaston Man  //                             . . + E E E - .
16644c9c1deSEaston Man  //                             . . + E E E E -
16744c9c1deSEaston Man  // As shown above, + means enqueue, - means dequeue, E is current content
16844c9c1deSEaston Man  // When dequeue, read port is organized like a banked FIFO
16944c9c1deSEaston Man  // Dequeue reads no more than 1 entry from each bank sequentially, this can be exploit to reduce area
17044c9c1deSEaston Man  // Enqueue writes cannot benefit from this characteristic unless use a SRAM
17144c9c1deSEaston Man  // For detail see Enqueue and Dequeue below
17244c9c1deSEaston Man  private val ibuf: Vec[IBufEntry] = RegInit(VecInit.fill(IBufSize)(0.U.asTypeOf(new IBufEntry)))
17344c9c1deSEaston Man  private val bankedIBufView: Vec[Vec[IBufEntry]] = VecInit.tabulate(IBufNBank)(
17444c9c1deSEaston Man    bankID => VecInit.tabulate(bankSize)(
17544c9c1deSEaston Man      inBankOffset => ibuf(bankID + inBankOffset * IBufNBank)
17644c9c1deSEaston Man    )
17744c9c1deSEaston Man  )
17844c9c1deSEaston Man
1798fae59bbSEaston Man
1808fae59bbSEaston Man  // Bypass wire
1818fae59bbSEaston Man  private val bypassEntries = WireDefault(VecInit.fill(DecodeWidth)(0.U.asTypeOf(Valid(new IBufEntry))))
1828fae59bbSEaston Man  // Normal read wire
1838fae59bbSEaston Man  private val deqEntries = WireDefault(VecInit.fill(DecodeWidth)(0.U.asTypeOf(Valid(new IBufEntry))))
1848fae59bbSEaston Man  // Output register
1858fae59bbSEaston Man  private val outputEntries = RegInit(VecInit.fill(DecodeWidth)(0.U.asTypeOf(Valid(new IBufEntry))))
186a5546049Sxiaofeibao  private val outputEntriesValidNum = PriorityMuxDefault(outputEntries.map(_.valid).zip(Seq.range(1, DecodeWidth).map(_.U)).reverse.toSeq, 0.U)
1878fae59bbSEaston Man
18844c9c1deSEaston Man  // Between Bank
18944c9c1deSEaston Man  private val deqBankPtrVec: Vec[IBufBankPtr] = RegInit(VecInit.tabulate(DecodeWidth)(_.U.asTypeOf(new IBufBankPtr)))
19044c9c1deSEaston Man  private val deqBankPtr: IBufBankPtr = deqBankPtrVec(0)
19105cc2a4eSXuan Hu  private val deqBankPtrVecNext = Wire(deqBankPtrVec.cloneType)
19244c9c1deSEaston Man  // Inside Bank
19344c9c1deSEaston Man  private val deqInBankPtr: Vec[IBufInBankPtr] = RegInit(VecInit.fill(IBufNBank)(0.U.asTypeOf(new IBufInBankPtr)))
19405cc2a4eSXuan Hu  private val deqInBankPtrNext = Wire(deqInBankPtr.cloneType)
19544c9c1deSEaston Man
19644c9c1deSEaston Man  val deqPtr = RegInit(0.U.asTypeOf(new IBufPtr))
19705cc2a4eSXuan Hu  val deqPtrNext = Wire(deqPtr.cloneType)
19844c9c1deSEaston Man
19944c9c1deSEaston Man  val enqPtrVec = RegInit(VecInit.tabulate(PredictWidth)(_.U.asTypeOf(new IBufPtr)))
20044c9c1deSEaston Man  val enqPtr = enqPtrVec(0)
20144c9c1deSEaston Man
2028fae59bbSEaston Man  val numTryEnq = WireDefault(0.U)
2038fae59bbSEaston Man  val numEnq = Mux(io.in.fire, numTryEnq, 0.U)
2048fae59bbSEaston Man
2058506cfc0Sxiaofeibao  // empty and decode can accept insts
2068506cfc0Sxiaofeibao  val useBypass = enqPtr === deqPtr && decodeCanAccept
20705cc2a4eSXuan Hu
20805cc2a4eSXuan Hu  // The number of decode accepted insts.
20905cc2a4eSXuan Hu  // Since decode promises accepting insts in order, use priority encoder to simplify the accumulation.
210a5546049Sxiaofeibao  private val numOut = Wire(UInt(log2Ceil(DecodeWidth).W))
211a5546049Sxiaofeibao  private val numDeq = numOut
21205cc2a4eSXuan Hu
21305cc2a4eSXuan Hu  // counter current number of valid
21405cc2a4eSXuan Hu  val numValid = distanceBetween(enqPtr, deqPtr)
21505cc2a4eSXuan Hu  val numValidAfterDeq = numValid - numDeq
21605cc2a4eSXuan Hu  // counter next number of valid
21705cc2a4eSXuan Hu  val numValidNext = numValid + numEnq - numDeq
21805cc2a4eSXuan Hu  val allowEnq = RegInit(true.B)
21905cc2a4eSXuan Hu  val numFromFetch = Mux(io.in.valid, PopCount(io.in.bits.enqEnable), 0.U)
22005cc2a4eSXuan Hu
22105cc2a4eSXuan Hu  allowEnq := (IBufSize - PredictWidth).U >= numValidNext // Disable when almost full
22244c9c1deSEaston Man
2238fae59bbSEaston Man  val enqOffset = VecInit.tabulate(PredictWidth)(i => PopCount(io.in.bits.valid.asBools.take(i)))
2248fae59bbSEaston Man  val enqData = VecInit.tabulate(PredictWidth)(i => Wire(new IBufEntry).fromFetch(io.in.bits, i))
2258fae59bbSEaston Man
226a5546049Sxiaofeibao  val outputEntriesIsNotFull = !outputEntries(DecodeWidth-1).valid
227a5546049Sxiaofeibao  when(decodeCanAccept) {
228a5546049Sxiaofeibao    numOut := Mux(numValid >= DecodeWidth.U, DecodeWidth.U, numValid)
229a5546049Sxiaofeibao  }.elsewhen(outputEntriesIsNotFull) {
230a5546049Sxiaofeibao    numOut := Mux(numValid >= DecodeWidth.U - outputEntriesValidNum, DecodeWidth.U - outputEntriesValidNum, numValid)
231a5546049Sxiaofeibao  }.otherwise {
232a5546049Sxiaofeibao    numOut := 0.U
233a5546049Sxiaofeibao  }
234a5546049Sxiaofeibao  val numBypass = Wire(UInt(log2Ceil(DecodeWidth).W))
2358fae59bbSEaston Man  // when using bypass, bypassed entries do not enqueue
2368fae59bbSEaston Man  when(useBypass) {
2378fae59bbSEaston Man    when(numFromFetch >= DecodeWidth.U) {
2388fae59bbSEaston Man      numTryEnq := numFromFetch - DecodeWidth.U
239a5546049Sxiaofeibao      numBypass := DecodeWidth.U
2408fae59bbSEaston Man    } .otherwise {
2418fae59bbSEaston Man      numTryEnq := 0.U
242a5546049Sxiaofeibao      numBypass := numFromFetch
2438fae59bbSEaston Man    }
2448fae59bbSEaston Man  } .otherwise {
2458fae59bbSEaston Man    numTryEnq := numFromFetch
246a5546049Sxiaofeibao    numBypass := 0.U
2478fae59bbSEaston Man  }
2488fae59bbSEaston Man
2498fae59bbSEaston Man  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2508fae59bbSEaston Man  // Bypass
2518fae59bbSEaston Man  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2528fae59bbSEaston Man  bypassEntries.zipWithIndex.foreach {
2538fae59bbSEaston Man    case (entry, idx) =>
2548fae59bbSEaston Man      // Select
2558fae59bbSEaston Man      val validOH = Range(0, PredictWidth).map {
2568fae59bbSEaston Man        i =>
2578fae59bbSEaston Man          io.in.bits.valid(i) &&
2588fae59bbSEaston Man            io.in.bits.enqEnable(i) &&
2598fae59bbSEaston Man            enqOffset(i) === idx.asUInt
2608fae59bbSEaston Man      } // Should be OneHot
2618fae59bbSEaston Man      entry.valid := validOH.reduce(_ || _) && io.in.fire && !io.flush
2628fae59bbSEaston Man      entry.bits := Mux1H(validOH, enqData)
2638fae59bbSEaston Man
2648fae59bbSEaston Man      // Debug Assertion
2659afa8a47STang Haojin      XSError(io.in.valid && PopCount(validOH) > 1.asUInt, "validOH is not OneHot")
2668fae59bbSEaston Man  }
2678fae59bbSEaston Man
2688fae59bbSEaston Man  // => Decode Output
2698fae59bbSEaston Man  // clean register output
2708fae59bbSEaston Man  io.out zip outputEntries foreach {
2718fae59bbSEaston Man    case (io, reg) =>
2728fae59bbSEaston Man      io.valid := reg.valid
2738fae59bbSEaston Man      io.bits := reg.bits.toCtrlFlow
2748fae59bbSEaston Man  }
275a5546049Sxiaofeibao  (outputEntries zip bypassEntries).zipWithIndex.foreach {
276a5546049Sxiaofeibao    case ((out, bypass), i) =>
27705cc2a4eSXuan Hu      when(decodeCanAccept) {
27805cc2a4eSXuan Hu        when(useBypass && io.in.valid) {
2798fae59bbSEaston Man          out := bypass
280e778bb8aSxiaofeibao-xjtu        }.otherwise {
281a5546049Sxiaofeibao          out := deqEntries(i)
2828fae59bbSEaston Man        }
283a5546049Sxiaofeibao      }.elsewhen(outputEntriesIsNotFull){
284a5546049Sxiaofeibao        out.valid := deqEntries(i).valid
285a5546049Sxiaofeibao        out.bits := Mux(i.U < outputEntriesValidNum, out.bits, VecInit(deqEntries.take(i + 1).map(_.bits))(i.U - outputEntriesValidNum))
2868fae59bbSEaston Man      }
2878fae59bbSEaston Man  }
2888fae59bbSEaston Man
28944c9c1deSEaston Man  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
29044c9c1deSEaston Man  // Enqueue
29144c9c1deSEaston Man  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
29244c9c1deSEaston Man  io.in.ready := allowEnq
29344c9c1deSEaston Man  // Data
29444c9c1deSEaston Man  ibuf.zipWithIndex.foreach {
29544c9c1deSEaston Man    case (entry, idx) => {
29644c9c1deSEaston Man      // Select
29744c9c1deSEaston Man      val validOH = Range(0, PredictWidth).map {
2988fae59bbSEaston Man        i =>
2998fae59bbSEaston Man          val useBypassMatch = enqOffset(i) >= DecodeWidth.U &&
3008fae59bbSEaston Man            enqPtrVec(enqOffset(i) - DecodeWidth.U).value === idx.asUInt
3018fae59bbSEaston Man          val normalMatch = enqPtrVec(enqOffset(i)).value === idx.asUInt
3028fae59bbSEaston Man          val m = Mux(useBypass, useBypassMatch, normalMatch) // when using bypass, bypassed entries do not enqueue
3038fae59bbSEaston Man
3048fae59bbSEaston Man          io.in.bits.valid(i) && io.in.bits.enqEnable(i) && m
30544c9c1deSEaston Man      } // Should be OneHot
30644c9c1deSEaston Man      val wen = validOH.reduce(_ || _) && io.in.fire && !io.flush
30744c9c1deSEaston Man
30844c9c1deSEaston Man      // Write port
30944c9c1deSEaston Man      // Each IBuffer entry has a PredictWidth -> 1 Mux
31044c9c1deSEaston Man      val writeEntry = Mux1H(validOH, enqData)
31144c9c1deSEaston Man      entry := Mux(wen, writeEntry, entry)
31244c9c1deSEaston Man
31344c9c1deSEaston Man      // Debug Assertion
31478c76c74STang Haojin      XSError(io.in.valid && PopCount(validOH) > 1.asUInt, "validOH is not OneHot")
31544c9c1deSEaston Man    }
31644c9c1deSEaston Man  }
31744c9c1deSEaston Man  // Pointer maintenance
31844c9c1deSEaston Man  when (io.in.fire && !io.flush) {
3198fae59bbSEaston Man    enqPtrVec := VecInit(enqPtrVec.map(_ + numTryEnq))
32044c9c1deSEaston Man  }
32144c9c1deSEaston Man
32244c9c1deSEaston Man  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
32344c9c1deSEaston Man  // Dequeue
32444c9c1deSEaston Man  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
325a5546049Sxiaofeibao  val outputEntriesValidNumNext = Wire(UInt(log2Ceil(DecodeWidth).W))
326a5546049Sxiaofeibao  XSError(outputEntriesValidNumNext > DecodeWidth.U, "Ibuffer: outputEntriesValidNumNext > DecodeWidth.U")
327a5546049Sxiaofeibao  val validVec = UIntToMask(outputEntriesValidNumNext(log2Ceil(DecodeWidth) - 1, 0), DecodeWidth)
328a5546049Sxiaofeibao  when(decodeCanAccept) {
329a5546049Sxiaofeibao    outputEntriesValidNumNext := Mux(useBypass, numBypass, numDeq)
330a5546049Sxiaofeibao  }.elsewhen(outputEntriesIsNotFull) {
331a5546049Sxiaofeibao    outputEntriesValidNumNext := outputEntriesValidNum + numDeq
332a5546049Sxiaofeibao  }.otherwise {
333a5546049Sxiaofeibao    outputEntriesValidNumNext := outputEntriesValidNum
334a5546049Sxiaofeibao  }
33544c9c1deSEaston Man  // Data
33644c9c1deSEaston Man  // Read port
33744c9c1deSEaston Man  // 2-stage, IBufNBank * (bankSize -> 1) + IBufNBank -> 1
33844c9c1deSEaston Man  // Should be better than IBufSize -> 1 in area, with no significant latency increase
33944c9c1deSEaston Man  private val readStage1: Vec[IBufEntry] = VecInit.tabulate(IBufNBank)(
340a5546049Sxiaofeibao    bankID => Mux1H(UIntToOH(deqInBankPtr(bankID).value), bankedIBufView(bankID))
34144c9c1deSEaston Man  )
34244c9c1deSEaston Man  for (i <- 0 until DecodeWidth) {
3438fae59bbSEaston Man    deqEntries(i).valid := validVec(i)
344a5546049Sxiaofeibao    deqEntries(i).bits := Mux1H(UIntToOH(deqBankPtrVec(i).value), readStage1)
34544c9c1deSEaston Man  }
34644c9c1deSEaston Man  // Pointer maintenance
347ac3c9508SXuan Hu  deqBankPtrVecNext := VecInit(deqBankPtrVec.map(_ + numDeq))
348ac3c9508SXuan Hu  deqPtrNext := deqPtr + numDeq
34905cc2a4eSXuan Hu  deqInBankPtrNext.zip(deqInBankPtr).zipWithIndex.foreach {
35005cc2a4eSXuan Hu    case ((ptrNext, ptr), idx) => {
35144c9c1deSEaston Man      // validVec[k] == bankValid[deqBankPtr + k]
35244c9c1deSEaston Man      // So bankValid[n] == validVec[n - deqBankPtr]
35344c9c1deSEaston Man      val validIdx = Mux(idx.asUInt >= deqBankPtr.value,
35444c9c1deSEaston Man        idx.asUInt - deqBankPtr.value,
35544c9c1deSEaston Man        ((idx + IBufNBank).asUInt - deqBankPtr.value)(log2Ceil(IBufNBank) - 1, 0)
35645b8fd86SEaston Man      )(log2Ceil(DecodeWidth) - 1, 0)
357a5546049Sxiaofeibao      val bankAdvance = numOut > validIdx
35805cc2a4eSXuan Hu      ptrNext := Mux(bankAdvance , ptr + 1.U, ptr)
35944c9c1deSEaston Man    }
36044c9c1deSEaston Man  }
36144c9c1deSEaston Man
36244c9c1deSEaston Man  // Flush
36344c9c1deSEaston Man  when (io.flush) {
36444c9c1deSEaston Man    allowEnq := true.B
36544c9c1deSEaston Man    enqPtrVec := enqPtrVec.indices.map(_.U.asTypeOf(new IBufPtr))
36644c9c1deSEaston Man    deqBankPtrVec := deqBankPtrVec.indices.map(_.U.asTypeOf(new IBufBankPtr))
36744c9c1deSEaston Man    deqInBankPtr := VecInit.fill(IBufNBank)(0.U.asTypeOf(new IBufInBankPtr))
36844c9c1deSEaston Man    deqPtr := 0.U.asTypeOf(new IBufPtr())
3698fae59bbSEaston Man    outputEntries.foreach(_.valid := false.B)
37005cc2a4eSXuan Hu  }.otherwise {
37105cc2a4eSXuan Hu    deqPtr := deqPtrNext
37205cc2a4eSXuan Hu    deqInBankPtr := deqInBankPtrNext
37305cc2a4eSXuan Hu    deqBankPtrVec := deqBankPtrVecNext
37444c9c1deSEaston Man  }
37544c9c1deSEaston Man  io.full := !allowEnq
37644c9c1deSEaston Man
37744c9c1deSEaston Man  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
37844c9c1deSEaston Man  // TopDown
37944c9c1deSEaston Man  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
38044c9c1deSEaston Man  val topdown_stage = RegInit(0.U.asTypeOf(new FrontendTopDownBundle))
38144c9c1deSEaston Man  topdown_stage := io.in.bits.topdown_info
38244c9c1deSEaston Man  when(io.flush) {
38344c9c1deSEaston Man    when(io.ControlRedirect) {
38444c9c1deSEaston Man      when(io.ControlBTBMissBubble) {
38544c9c1deSEaston Man        topdown_stage.reasons(TopDownCounters.BTBMissBubble.id) := true.B
38644c9c1deSEaston Man      }.elsewhen(io.TAGEMissBubble) {
38744c9c1deSEaston Man        topdown_stage.reasons(TopDownCounters.TAGEMissBubble.id) := true.B
38844c9c1deSEaston Man      }.elsewhen(io.SCMissBubble) {
38944c9c1deSEaston Man        topdown_stage.reasons(TopDownCounters.SCMissBubble.id) := true.B
39044c9c1deSEaston Man      }.elsewhen(io.ITTAGEMissBubble) {
39144c9c1deSEaston Man        topdown_stage.reasons(TopDownCounters.ITTAGEMissBubble.id) := true.B
39244c9c1deSEaston Man      }.elsewhen(io.RASMissBubble) {
39344c9c1deSEaston Man        topdown_stage.reasons(TopDownCounters.RASMissBubble.id) := true.B
39444c9c1deSEaston Man      }
39544c9c1deSEaston Man    }.elsewhen(io.MemVioRedirect) {
39644c9c1deSEaston Man      topdown_stage.reasons(TopDownCounters.MemVioRedirectBubble.id) := true.B
39744c9c1deSEaston Man    }.otherwise {
39844c9c1deSEaston Man      topdown_stage.reasons(TopDownCounters.OtherRedirectBubble.id) := true.B
39944c9c1deSEaston Man    }
40044c9c1deSEaston Man  }
40144c9c1deSEaston Man
40244c9c1deSEaston Man
40344c9c1deSEaston Man  val dequeueInsufficient = Wire(Bool())
40444c9c1deSEaston Man  val matchBubble = Wire(UInt(log2Up(TopDownCounters.NumStallReasons.id).W))
40544c9c1deSEaston Man  val deqValidCount = PopCount(validVec.asBools)
40644c9c1deSEaston Man  val deqWasteCount = DecodeWidth.U - deqValidCount
40744c9c1deSEaston Man  dequeueInsufficient := deqValidCount < DecodeWidth.U
40844c9c1deSEaston Man  matchBubble := (TopDownCounters.NumStallReasons.id - 1).U - PriorityEncoder(topdown_stage.reasons.reverse)
40944c9c1deSEaston Man
41044c9c1deSEaston Man  io.stallReason.reason.map(_ := 0.U)
41144c9c1deSEaston Man  for (i <- 0 until DecodeWidth) {
41244c9c1deSEaston Man    when(i.U < deqWasteCount) {
41344c9c1deSEaston Man      io.stallReason.reason(DecodeWidth - i - 1) := matchBubble
41444c9c1deSEaston Man    }
41544c9c1deSEaston Man  }
41644c9c1deSEaston Man
41744c9c1deSEaston Man  when(!(deqWasteCount === DecodeWidth.U || topdown_stage.reasons.asUInt.orR)) {
41844c9c1deSEaston Man    // should set reason for FetchFragmentationStall
41944c9c1deSEaston Man    // topdown_stage.reasons(TopDownCounters.FetchFragmentationStall.id) := true.B
42044c9c1deSEaston Man    for (i <- 0 until DecodeWidth) {
42144c9c1deSEaston Man      when(i.U < deqWasteCount) {
42244c9c1deSEaston Man        io.stallReason.reason(DecodeWidth - i - 1) := TopDownCounters.FetchFragBubble.id.U
42344c9c1deSEaston Man      }
42444c9c1deSEaston Man    }
42544c9c1deSEaston Man  }
42644c9c1deSEaston Man
42744c9c1deSEaston Man  when(io.stallReason.backReason.valid) {
42844c9c1deSEaston Man    io.stallReason.reason.map(_ := io.stallReason.backReason.bits)
42944c9c1deSEaston Man  }
43044c9c1deSEaston Man
43144c9c1deSEaston Man  // Debug info
43244c9c1deSEaston Man  XSError(
43344c9c1deSEaston Man    deqPtr.value =/= deqBankPtr.value + deqInBankPtr(deqBankPtr.value).value * IBufNBank.asUInt,
43444c9c1deSEaston Man    "Dequeue PTR mismatch"
43544c9c1deSEaston Man  )
43605cc2a4eSXuan Hu  XSError(isBefore(enqPtr, deqPtr) && !isFull(enqPtr, deqPtr), "\ndeqPtr is older than enqPtr!\n")
43705cc2a4eSXuan Hu
43844c9c1deSEaston Man  XSDebug(io.flush, "IBuffer Flushed\n")
43944c9c1deSEaston Man
44044c9c1deSEaston Man  when(io.in.fire) {
44144c9c1deSEaston Man    XSDebug("Enque:\n")
44244c9c1deSEaston Man    XSDebug(p"MASK=${Binary(io.in.bits.valid)}\n")
44344c9c1deSEaston Man    for(i <- 0 until PredictWidth){
44444c9c1deSEaston Man      XSDebug(p"PC=${Hexadecimal(io.in.bits.pc(i))} ${Hexadecimal(io.in.bits.instrs(i))}\n")
44544c9c1deSEaston Man    }
44644c9c1deSEaston Man  }
44744c9c1deSEaston Man
44844c9c1deSEaston Man  for (i <- 0 until DecodeWidth) {
44944c9c1deSEaston Man    XSDebug(io.out(i).fire,
45044c9c1deSEaston Man      p"deq: ${Hexadecimal(io.out(i).bits.instr)} PC=${Hexadecimal(io.out(i).bits.pc)}" +
45144c9c1deSEaston Man      p"v=${io.out(i).valid} r=${io.out(i).ready} " +
45244c9c1deSEaston Man      p"excpVec=${Binary(io.out(i).bits.exceptionVec.asUInt)} crossPageIPF=${io.out(i).bits.crossPageIPFFix}\n")
45344c9c1deSEaston Man  }
45444c9c1deSEaston Man
45505cc2a4eSXuan Hu  XSDebug(p"numValid: ${numValid}\n")
45644c9c1deSEaston Man  XSDebug(p"EnqNum: ${numEnq}\n")
45744c9c1deSEaston Man  XSDebug(p"DeqNum: ${numDeq}\n")
45844c9c1deSEaston Man
45944c9c1deSEaston Man  val afterInit = RegInit(false.B)
46044c9c1deSEaston Man  val headBubble = RegInit(false.B)
46144c9c1deSEaston Man  when (io.in.fire) { afterInit := true.B }
46244c9c1deSEaston Man  when (io.flush) {
46344c9c1deSEaston Man    headBubble := true.B
46405cc2a4eSXuan Hu  } .elsewhen(numValid =/= 0.U) {
46544c9c1deSEaston Man    headBubble := false.B
46644c9c1deSEaston Man  }
46705cc2a4eSXuan Hu  val instrHungry = afterInit && (numValid === 0.U) && !headBubble
46844c9c1deSEaston Man
46905cc2a4eSXuan Hu  QueuePerf(IBufSize, numValid, !allowEnq)
47044c9c1deSEaston Man  XSPerfAccumulate("flush", io.flush)
47144c9c1deSEaston Man  XSPerfAccumulate("hungry", instrHungry)
47244c9c1deSEaston Man
47305cc2a4eSXuan Hu  val ibuffer_IDWidth_hvButNotFull = afterInit && (numValid =/= 0.U) && (numValid < DecodeWidth.U) && !headBubble
47444c9c1deSEaston Man  XSPerfAccumulate("ibuffer_IDWidth_hvButNotFull", ibuffer_IDWidth_hvButNotFull)
47544c9c1deSEaston Man  /*
47644c9c1deSEaston Man  XSPerfAccumulate("ICacheMissBubble", Mux(matchBubbleVec(TopDownCounters.ICacheMissBubble.id), deqWasteCount, 0.U))
47744c9c1deSEaston Man  XSPerfAccumulate("ITLBMissBubble", Mux(matchBubbleVec(TopDownCounters.ITLBMissBubble.id), deqWasteCount, 0.U))
47844c9c1deSEaston Man  XSPerfAccumulate("ControlRedirectBubble", Mux(matchBubbleVec(TopDownCounters.ControlRedirectBubble.id), deqWasteCount, 0.U))
47944c9c1deSEaston Man  XSPerfAccumulate("MemVioRedirectBubble", Mux(matchBubbleVec(TopDownCounters.MemVioRedirectBubble.id), deqWasteCount, 0.U))
48044c9c1deSEaston Man  XSPerfAccumulate("OtherRedirectBubble", Mux(matchBubbleVec(TopDownCounters.OtherRedirectBubble.id), deqWasteCount, 0.U))
48144c9c1deSEaston Man  XSPerfAccumulate("BTBMissBubble", Mux(matchBubbleVec(TopDownCounters.BTBMissBubble.id), deqWasteCount, 0.U))
48244c9c1deSEaston Man  XSPerfAccumulate("OverrideBubble", Mux(matchBubbleVec(TopDownCounters.OverrideBubble.id), deqWasteCount, 0.U))
48344c9c1deSEaston Man  XSPerfAccumulate("FtqUpdateBubble", Mux(matchBubbleVec(TopDownCounters.FtqUpdateBubble.id), deqWasteCount, 0.U))
48444c9c1deSEaston Man  XSPerfAccumulate("FtqFullStall", Mux(matchBubbleVec(TopDownCounters.FtqFullStall.id), deqWasteCount, 0.U))
48544c9c1deSEaston Man  XSPerfAccumulate("FetchFragmentBubble",
48644c9c1deSEaston Man  Mux(deqWasteCount === DecodeWidth.U || topdown_stage.reasons.asUInt.orR, 0.U, deqWasteCount))
48744c9c1deSEaston Man  XSPerfAccumulate("TAGEMissBubble", Mux(matchBubbleVec(TopDownCounters.TAGEMissBubble.id), deqWasteCount, 0.U))
48844c9c1deSEaston Man  XSPerfAccumulate("SCMissBubble", Mux(matchBubbleVec(TopDownCounters.SCMissBubble.id), deqWasteCount, 0.U))
48944c9c1deSEaston Man  XSPerfAccumulate("ITTAGEMissBubble", Mux(matchBubbleVec(TopDownCounters.ITTAGEMissBubble.id), deqWasteCount, 0.U))
49044c9c1deSEaston Man  XSPerfAccumulate("RASMissBubble", Mux(matchBubbleVec(TopDownCounters.RASMissBubble.id), deqWasteCount, 0.U))
49144c9c1deSEaston Man  */
49244c9c1deSEaston Man
49344c9c1deSEaston Man  val perfEvents = Seq(
49444c9c1deSEaston Man    ("IBuffer_Flushed  ", io.flush                                                                     ),
49544c9c1deSEaston Man    ("IBuffer_hungry   ", instrHungry                                                                  ),
49605cc2a4eSXuan Hu    ("IBuffer_1_4_valid", (numValid >  (0*(IBufSize/4)).U) & (numValid < (1*(IBufSize/4)).U)   ),
49705cc2a4eSXuan Hu    ("IBuffer_2_4_valid", (numValid >= (1*(IBufSize/4)).U) & (numValid < (2*(IBufSize/4)).U)   ),
49805cc2a4eSXuan Hu    ("IBuffer_3_4_valid", (numValid >= (2*(IBufSize/4)).U) & (numValid < (3*(IBufSize/4)).U)   ),
49905cc2a4eSXuan Hu    ("IBuffer_4_4_valid", (numValid >= (3*(IBufSize/4)).U) & (numValid < (4*(IBufSize/4)).U)   ),
50005cc2a4eSXuan Hu    ("IBuffer_full     ",  numValid.andR                                                           ),
50144c9c1deSEaston Man    ("Front_Bubble     ", PopCount((0 until DecodeWidth).map(i => io.out(i).ready && !io.out(i).valid)))
50244c9c1deSEaston Man  )
50344c9c1deSEaston Man  generatePerfEvent()
50444c9c1deSEaston Man}
505