xref: /XiangShan/src/main/scala/xiangshan/mem/lsqueue/LoadQueueRAW.scala (revision 3c808de005aba6d7539d33be9962c44375b97e6d)
1e4f69d78Ssfencevma/***************************************************************************************
2e4f69d78Ssfencevma* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
3e4f69d78Ssfencevma* Copyright (c) 2020-2021 Peng Cheng Laboratory
4e4f69d78Ssfencevma*
5e4f69d78Ssfencevma* XiangShan is licensed under Mulan PSL v2.
6e4f69d78Ssfencevma* You can use this software according to the terms and conditions of the Mulan PSL v2.
7e4f69d78Ssfencevma* You may obtain a copy of Mulan PSL v2 at:
8e4f69d78Ssfencevma*          http://license.coscl.org.cn/MulanPSL2
9e4f69d78Ssfencevma*
10e4f69d78Ssfencevma* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
11e4f69d78Ssfencevma* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
12e4f69d78Ssfencevma* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
13e4f69d78Ssfencevma*
14e4f69d78Ssfencevma* See the Mulan PSL v2 for more details.
15e4f69d78Ssfencevma***************************************************************************************/
16e4f69d78Ssfencevma
17e4f69d78Ssfencevmapackage xiangshan.mem
18e4f69d78Ssfencevma
199e12e8edScz4eimport org.chipsalliance.cde.config._
20e4f69d78Ssfencevmaimport chisel3._
21e4f69d78Ssfencevmaimport chisel3.util._
22e4f69d78Ssfencevmaimport utils._
23e4f69d78Ssfencevmaimport utility._
249e12e8edScz4eimport xiangshan._
259e12e8edScz4eimport xiangshan.frontend.FtqPtr
269e12e8edScz4eimport xiangshan.backend.rob.RobPtr
27dfb4c5dcSXuan Huimport xiangshan.backend.Bundles.DynInst
289e12e8edScz4eimport xiangshan.mem.mdp._
299e12e8edScz4eimport xiangshan.mem.Bundles._
309e12e8edScz4eimport xiangshan.cache._
31e4f69d78Ssfencevma
32e4f69d78Ssfencevmaclass LoadQueueRAW(implicit p: Parameters) extends XSModule
33e4f69d78Ssfencevma  with HasDCacheParameters
34e4f69d78Ssfencevma  with HasCircularQueuePtrHelper
35e4f69d78Ssfencevma  with HasLoadHelper
36e4f69d78Ssfencevma  with HasPerfEvents
37e4f69d78Ssfencevma{
38e4f69d78Ssfencevma  val io = IO(new Bundle() {
3914a67055Ssfencevma    // control
40e4f69d78Ssfencevma    val redirect = Flipped(ValidIO(new Redirect))
4114a67055Ssfencevma
4214a67055Ssfencevma    // violation query
4314a67055Ssfencevma    val query = Vec(LoadPipelineWidth, Flipped(new LoadNukeQueryIO))
4414a67055Ssfencevma
4514a67055Ssfencevma    // from store unit s1
46e4f69d78Ssfencevma    val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle)))
4714a67055Ssfencevma
4814a67055Ssfencevma    // global rollback flush
4916ede6bbSweiding liu    val rollback = Vec(StorePipelineWidth,Output(Valid(new Redirect)))
5014a67055Ssfencevma
5114a67055Ssfencevma    // to LoadQueueReplay
52e4f69d78Ssfencevma    val stAddrReadySqPtr = Input(new SqPtr)
53e4f69d78Ssfencevma    val stIssuePtr       = Input(new SqPtr)
54e4f69d78Ssfencevma    val lqFull           = Output(Bool())
55e4f69d78Ssfencevma  })
56e4f69d78Ssfencevma
57549073a0Scz4e  private def PartialPAddrWidth: Int = 24
58*3c808de0SAnzo  private def paddrOffset: Int = DCacheVWordOffset
59549073a0Scz4e  private def genPartialPAddr(paddr: UInt) = {
60*3c808de0SAnzo    paddr(DCacheVWordOffset + PartialPAddrWidth - 1, paddrOffset)
61549073a0Scz4e  }
62549073a0Scz4e
63e4f69d78Ssfencevma  println("LoadQueueRAW: size " + LoadQueueRAWSize)
64e4f69d78Ssfencevma  //  LoadQueueRAW field
65e4f69d78Ssfencevma  //  +-------+--------+-------+-------+-----------+
66e4f69d78Ssfencevma  //  | Valid |  uop   |PAddr  | Mask  | Datavalid |
67e4f69d78Ssfencevma  //  +-------+--------+-------+-------+-----------+
68e4f69d78Ssfencevma  //
69e4f69d78Ssfencevma  //  Field descriptions:
70e4f69d78Ssfencevma  //  Allocated   : entry has been allocated already
71e4f69d78Ssfencevma  //  MicroOp     : inst's microOp
72e4f69d78Ssfencevma  //  PAddr       : physical address.
73e4f69d78Ssfencevma  //  Mask        : data mask
74e4f69d78Ssfencevma  //  Datavalid   : data valid
75e4f69d78Ssfencevma  //
76e4f69d78Ssfencevma  val allocated = RegInit(VecInit(List.fill(LoadQueueRAWSize)(false.B))) // The control signals need to explicitly indicate the initial value
77dfb4c5dcSXuan Hu  val uop = Reg(Vec(LoadQueueRAWSize, new DynInst))
78e4f69d78Ssfencevma  val paddrModule = Module(new LqPAddrModule(
79549073a0Scz4e    gen = UInt(PartialPAddrWidth.W),
80e4f69d78Ssfencevma    numEntries = LoadQueueRAWSize,
81e4f69d78Ssfencevma    numRead = LoadPipelineWidth,
82e4f69d78Ssfencevma    numWrite = LoadPipelineWidth,
83e4f69d78Ssfencevma    numWBank = LoadQueueNWriteBanks,
84e4f69d78Ssfencevma    numWDelay = 2,
85*3c808de0SAnzo    numCamPort = StorePipelineWidth,
86*3c808de0SAnzo    enableCacheLineCheck = true,
87*3c808de0SAnzo    paddrOffset = paddrOffset
88e4f69d78Ssfencevma  ))
89e4f69d78Ssfencevma  paddrModule.io := DontCare
90e4f69d78Ssfencevma  val maskModule = Module(new LqMaskModule(
91cdbff57cSHaoyuan Feng    gen = UInt((VLEN/8).W),
92e4f69d78Ssfencevma    numEntries = LoadQueueRAWSize,
93e4f69d78Ssfencevma    numRead = LoadPipelineWidth,
94e4f69d78Ssfencevma    numWrite = LoadPipelineWidth,
95e4f69d78Ssfencevma    numWBank = LoadQueueNWriteBanks,
96e4f69d78Ssfencevma    numWDelay = 2,
97e4f69d78Ssfencevma    numCamPort = StorePipelineWidth
98e4f69d78Ssfencevma  ))
99e4f69d78Ssfencevma  maskModule.io := DontCare
100e4f69d78Ssfencevma  val datavalid = RegInit(VecInit(List.fill(LoadQueueRAWSize)(false.B)))
101e4f69d78Ssfencevma
102e4f69d78Ssfencevma  // freeliset: store valid entries index.
103e4f69d78Ssfencevma  // +---+---+--------------+-----+-----+
104e4f69d78Ssfencevma  // | 0 | 1 |      ......  | n-2 | n-1 |
105e4f69d78Ssfencevma  // +---+---+--------------+-----+-----+
106e4f69d78Ssfencevma  val freeList = Module(new FreeList(
107e4f69d78Ssfencevma    size = LoadQueueRAWSize,
108e4f69d78Ssfencevma    allocWidth = LoadPipelineWidth,
109e4f69d78Ssfencevma    freeWidth = 4,
110f275998aSsfencevma    enablePreAlloc = true,
111e4f69d78Ssfencevma    moduleName = "LoadQueueRAW freelist"
112e4f69d78Ssfencevma  ))
113e4f69d78Ssfencevma  freeList.io := DontCare
114e4f69d78Ssfencevma
115e4f69d78Ssfencevma  //  LoadQueueRAW enqueue
116e4f69d78Ssfencevma  val canEnqueue = io.query.map(_.req.valid)
117e4f69d78Ssfencevma  val cancelEnqueue = io.query.map(_.req.bits.uop.robIdx.needFlush(io.redirect))
118e4f69d78Ssfencevma  val allAddrCheck = io.stIssuePtr === io.stAddrReadySqPtr
119e4f69d78Ssfencevma  val hasAddrInvalidStore = io.query.map(_.req.bits.uop.sqIdx).map(sqIdx => {
120e4f69d78Ssfencevma    Mux(!allAddrCheck, isBefore(io.stAddrReadySqPtr, sqIdx), false.B)
121e4f69d78Ssfencevma  })
122e4f69d78Ssfencevma  val needEnqueue = canEnqueue.zip(hasAddrInvalidStore).zip(cancelEnqueue).map { case ((v, r), c) => v && r && !c }
123e4f69d78Ssfencevma
124e4f69d78Ssfencevma  // Allocate logic
125f275998aSsfencevma  val acceptedVec = Wire(Vec(LoadPipelineWidth, Bool()))
12631fae68eSYanqin Li  val enqIndexVec = Wire(Vec(LoadPipelineWidth, UInt(log2Up(LoadQueueRAWSize).W)))
127e4f69d78Ssfencevma
128e4f69d78Ssfencevma  // Enqueue
129e4f69d78Ssfencevma  for ((enq, w) <- io.query.map(_.req).zipWithIndex) {
130f275998aSsfencevma    acceptedVec(w) := false.B
131e4f69d78Ssfencevma    paddrModule.io.wen(w) := false.B
132e4f69d78Ssfencevma    maskModule.io.wen(w) := false.B
133e4f69d78Ssfencevma    freeList.io.doAllocate(w) := false.B
134e4f69d78Ssfencevma
135f275998aSsfencevma    freeList.io.allocateReq(w) := true.B
136e4f69d78Ssfencevma
137e4f69d78Ssfencevma    //  Allocate ready
138f275998aSsfencevma    val offset = PopCount(needEnqueue.take(w))
139f275998aSsfencevma    val canAccept = freeList.io.canAllocate(offset)
140f275998aSsfencevma    val enqIndex = freeList.io.allocateSlot(offset)
141f275998aSsfencevma    enq.ready := Mux(needEnqueue(w), canAccept, true.B)
142e4f69d78Ssfencevma
143f275998aSsfencevma    enqIndexVec(w) := enqIndex
144e4f69d78Ssfencevma    when (needEnqueue(w) && enq.ready) {
145f275998aSsfencevma      acceptedVec(w) := true.B
146f275998aSsfencevma
147e4f69d78Ssfencevma      freeList.io.doAllocate(w) := true.B
148e4f69d78Ssfencevma
149e4f69d78Ssfencevma      //  Allocate new entry
150e4f69d78Ssfencevma      allocated(enqIndex) := true.B
151e4f69d78Ssfencevma
152e4f69d78Ssfencevma      //  Write paddr
153e4f69d78Ssfencevma      paddrModule.io.wen(w) := true.B
154e4f69d78Ssfencevma      paddrModule.io.waddr(w) := enqIndex
155549073a0Scz4e      paddrModule.io.wdata(w) := genPartialPAddr(enq.bits.paddr)
156e4f69d78Ssfencevma
157e4f69d78Ssfencevma      //  Write mask
158e4f69d78Ssfencevma      maskModule.io.wen(w) := true.B
159e4f69d78Ssfencevma      maskModule.io.waddr(w) := enqIndex
160e4f69d78Ssfencevma      maskModule.io.wdata(w) := enq.bits.mask
161e4f69d78Ssfencevma
162e4f69d78Ssfencevma      //  Fill info
163e4f69d78Ssfencevma      uop(enqIndex) := enq.bits.uop
16414a67055Ssfencevma      datavalid(enqIndex) := enq.bits.data_valid
165e4f69d78Ssfencevma    }
1668b33cd30Sklin02    val debug_robIdx = enq.bits.uop.robIdx.asUInt
1678b33cd30Sklin02    XSError(needEnqueue(w) && enq.ready && allocated(enqIndex), p"LoadQueueRAW: You can not write an valid entry! check: ldu $w, robIdx $debug_robIdx")
168e4f69d78Ssfencevma  }
169e4f69d78Ssfencevma
170e4f69d78Ssfencevma  for ((query, w) <- io.query.map(_.resp).zipWithIndex) {
171e4f69d78Ssfencevma    query.valid := RegNext(io.query(w).req.valid)
17214a67055Ssfencevma    query.bits.rep_frm_fetch := RegNext(false.B)
173e4f69d78Ssfencevma  }
174e4f69d78Ssfencevma
175e4f69d78Ssfencevma  //  LoadQueueRAW deallocate
176e4f69d78Ssfencevma  val freeMaskVec = Wire(Vec(LoadQueueRAWSize, Bool()))
177e4f69d78Ssfencevma
178e4f69d78Ssfencevma  // init
179e4f69d78Ssfencevma  freeMaskVec.map(e => e := false.B)
180e4f69d78Ssfencevma
181e4f69d78Ssfencevma  // when the stores that "older than" current load address were ready.
182e4f69d78Ssfencevma  // current load will be released.
183e4f69d78Ssfencevma  for (i <- 0 until LoadQueueRAWSize) {
184e4f69d78Ssfencevma    val deqNotBlock = Mux(!allAddrCheck, !isBefore(io.stAddrReadySqPtr, uop(i).sqIdx), true.B)
185e4f69d78Ssfencevma    val needCancel = uop(i).robIdx.needFlush(io.redirect)
186e4f69d78Ssfencevma
1879b12a106SAnzo    when (allocated(i) && (deqNotBlock || needCancel)) {
188e4f69d78Ssfencevma      allocated(i) := false.B
189e4f69d78Ssfencevma      freeMaskVec(i) := true.B
190e4f69d78Ssfencevma    }
191e4f69d78Ssfencevma  }
192e4f69d78Ssfencevma
193e4f69d78Ssfencevma  // if need replay deallocate entry
1945003e6f8SHuijin Li  val lastCanAccept = GatedValidRegNext(acceptedVec)
1955003e6f8SHuijin Li  val lastAllocIndex = GatedRegNext(enqIndexVec)
196e4f69d78Ssfencevma
19714a67055Ssfencevma  for ((revoke, w) <- io.query.map(_.revoke).zipWithIndex) {
19814a67055Ssfencevma    val revokeValid = revoke && lastCanAccept(w)
19914a67055Ssfencevma    val revokeIndex = lastAllocIndex(w)
200e4f69d78Ssfencevma
20114a67055Ssfencevma    when (allocated(revokeIndex) && revokeValid) {
20214a67055Ssfencevma      allocated(revokeIndex) := false.B
20314a67055Ssfencevma      freeMaskVec(revokeIndex) := true.B
204e4f69d78Ssfencevma    }
205e4f69d78Ssfencevma  }
206e4f69d78Ssfencevma  freeList.io.free := freeMaskVec.asUInt
207e4f69d78Ssfencevma
208e4f69d78Ssfencevma  io.lqFull := freeList.io.empty
209e4f69d78Ssfencevma
210e4f69d78Ssfencevma  /**
211e4f69d78Ssfencevma    * Store-Load Memory violation detection
212e4f69d78Ssfencevma    * Scheme 1(Current scheme): flush the pipeline then re-fetch from the load instruction (like old load queue).
213e4f69d78Ssfencevma    * Scheme 2                : re-fetch instructions from the first instruction after the store instruction.
214e4f69d78Ssfencevma    *
215e4f69d78Ssfencevma    * When store writes back, it searches LoadQueue for younger load instructions
216e4f69d78Ssfencevma    * with the same load physical address. They loaded wrong data and need re-execution.
217e4f69d78Ssfencevma    *
218e4f69d78Ssfencevma    * Cycle 0: Store Writeback
219e4f69d78Ssfencevma    *   Generate match vector for store address with rangeMask(stPtr, enqPtr).
220e4f69d78Ssfencevma    * Cycle 1: Select oldest load from select group.
221e4f69d78Ssfencevma    * Cycle x: Redirect Fire
222e4f69d78Ssfencevma    *   Choose the oldest load from LoadPipelineWidth oldest loads.
223e4f69d78Ssfencevma    *   Prepare redirect request according to the detected violation.
224e4f69d78Ssfencevma    *   Fire redirect request (if valid)
225e4f69d78Ssfencevma    */
226e4f69d78Ssfencevma  //              SelectGroup 0         SelectGroup 1          SelectGroup y
227e4f69d78Ssfencevma  // stage 0:       lq  lq  lq  ......    lq  lq  lq  .......    lq  lq  lq
228e4f69d78Ssfencevma  //                |   |   |             |   |   |              |   |   |
229e4f69d78Ssfencevma  // stage 1:       lq  lq  lq  ......    lq  lq  lq  .......    lq  lq  lq
230e4f69d78Ssfencevma  //                 \  |  /    ......     \  |  /    .......     \  |  /
231e4f69d78Ssfencevma  // stage 2:           lq                    lq                     lq
232e4f69d78Ssfencevma  //                     \  |  /  .......  \  |  /   ........  \  |  /
233e4f69d78Ssfencevma  // stage 3:               lq                lq                  lq
234e4f69d78Ssfencevma  //                                          ...
235e4f69d78Ssfencevma  //                                          ...
236e4f69d78Ssfencevma  //                                           |
237e4f69d78Ssfencevma  // stage x:                                  lq
238e4f69d78Ssfencevma  //                                           |
239e4f69d78Ssfencevma  //                                       rollback req
240e4f69d78Ssfencevma
241e4f69d78Ssfencevma  // select logic
242e4f69d78Ssfencevma  val SelectGroupSize = RollbackGroupSize
243e4f69d78Ssfencevma  val lgSelectGroupSize = log2Ceil(SelectGroupSize)
244e4f69d78Ssfencevma  val TotalSelectCycles = scala.math.ceil(log2Ceil(LoadQueueRAWSize).toFloat / lgSelectGroupSize).toInt + 1
245e4f69d78Ssfencevma
246e4f69d78Ssfencevma  def selectPartialOldest[T <: XSBundleWithMicroOp](valid: Seq[Bool], bits: Seq[T]): (Seq[Bool], Seq[T]) = {
247e4f69d78Ssfencevma    assert(valid.length == bits.length)
248e4f69d78Ssfencevma    if (valid.length == 0 || valid.length == 1) {
249e4f69d78Ssfencevma      (valid, bits)
250e4f69d78Ssfencevma    } else if (valid.length == 2) {
251e4f69d78Ssfencevma      val res = Seq.fill(2)(Wire(ValidIO(chiselTypeOf(bits(0)))))
252e4f69d78Ssfencevma      for (i <- res.indices) {
253e4f69d78Ssfencevma        res(i).valid := valid(i)
254e4f69d78Ssfencevma        res(i).bits := bits(i)
255e4f69d78Ssfencevma      }
256e4f69d78Ssfencevma      val oldest = Mux(valid(0) && valid(1), Mux(isAfter(bits(0).uop.robIdx, bits(1).uop.robIdx), res(1), res(0)), Mux(valid(0) && !valid(1), res(0), res(1)))
257e4f69d78Ssfencevma      (Seq(oldest.valid), Seq(oldest.bits))
258e4f69d78Ssfencevma    } else {
259e4f69d78Ssfencevma      val left = selectPartialOldest(valid.take(valid.length / 2), bits.take(bits.length / 2))
260e4f69d78Ssfencevma      val right = selectPartialOldest(valid.takeRight(valid.length - (valid.length / 2)), bits.takeRight(bits.length - (bits.length / 2)))
261e4f69d78Ssfencevma      selectPartialOldest(left._1 ++ right._1, left._2 ++ right._2)
262e4f69d78Ssfencevma    }
263e4f69d78Ssfencevma  }
264e4f69d78Ssfencevma
265e4f69d78Ssfencevma  def selectOldest[T <: XSBundleWithMicroOp](valid: Seq[Bool], bits: Seq[T]): (Seq[Bool], Seq[T]) = {
266e4f69d78Ssfencevma    assert(valid.length == bits.length)
267e4f69d78Ssfencevma    val numSelectGroups = scala.math.ceil(valid.length.toFloat / SelectGroupSize).toInt
268e4f69d78Ssfencevma
269e4f69d78Ssfencevma    // group info
270660213bdSsfencevma    val selectValidGroups = valid.grouped(SelectGroupSize).toList
271660213bdSsfencevma    val selectBitsGroups = bits.grouped(SelectGroupSize).toList
272e4f69d78Ssfencevma    // select logic
273e4f69d78Ssfencevma    if (valid.length <= SelectGroupSize) {
274e4f69d78Ssfencevma      val (selValid, selBits) = selectPartialOldest(valid, bits)
2755003e6f8SHuijin Li      val selValidNext = GatedValidRegNext(selValid(0))
2765003e6f8SHuijin Li      val selBitsNext = RegEnable(selBits(0), selValid(0))
27716ede6bbSweiding liu      (Seq(selValidNext && !selBitsNext.uop.robIdx.needFlush(RegNext(io.redirect))), Seq(selBitsNext))
278e4f69d78Ssfencevma    } else {
279e4f69d78Ssfencevma      val select = (0 until numSelectGroups).map(g => {
280e4f69d78Ssfencevma        val (selValid, selBits) = selectPartialOldest(selectValidGroups(g), selectBitsGroups(g))
281f275998aSsfencevma        val selValidNext = RegNext(selValid(0))
2825003e6f8SHuijin Li        val selBitsNext = RegEnable(selBits(0), selValid(0))
283f275998aSsfencevma        (selValidNext && !selBitsNext.uop.robIdx.needFlush(io.redirect) && !selBitsNext.uop.robIdx.needFlush(RegNext(io.redirect)), selBitsNext)
284e4f69d78Ssfencevma      })
285e4f69d78Ssfencevma      selectOldest(select.map(_._1), select.map(_._2))
286e4f69d78Ssfencevma    }
287e4f69d78Ssfencevma  }
288e4f69d78Ssfencevma
28926af847eSgood-circle  val storeIn = io.storeIn
290e4f69d78Ssfencevma
29120a5248fSzhanglinjuan  def detectRollback(i: Int) = {
292549073a0Scz4e    paddrModule.io.violationMdata(i) := genPartialPAddr(RegEnable(storeIn(i).bits.paddr, storeIn(i).valid))
293*3c808de0SAnzo    paddrModule.io.violationCheckLine.get(i) := storeIn(i).bits.wlineflag
2945003e6f8SHuijin Li    maskModule.io.violationMdata(i) := RegEnable(storeIn(i).bits.mask, storeIn(i).valid)
29520a5248fSzhanglinjuan
296aebc38d1Ssfencevma    val addrMaskMatch = paddrModule.io.violationMmask(i).asUInt & maskModule.io.violationMmask(i).asUInt
2975003e6f8SHuijin Li    val entryNeedCheck = GatedValidRegNext(VecInit((0 until LoadQueueRAWSize).map(j => {
298520ec528SXuan Hu      allocated(j) && storeIn(i).valid && isAfter(uop(j).robIdx, storeIn(i).bits.uop.robIdx) && datavalid(j) && !uop(j).robIdx.needFlush(io.redirect)
299e4f69d78Ssfencevma    })))
300e4f69d78Ssfencevma    val lqViolationSelVec = VecInit((0 until LoadQueueRAWSize).map(j => {
301e4f69d78Ssfencevma      addrMaskMatch(j) && entryNeedCheck(j)
302e4f69d78Ssfencevma    }))
303e4f69d78Ssfencevma
304e4f69d78Ssfencevma    val lqViolationSelUopExts = uop.map(uop => {
305e4f69d78Ssfencevma      val wrapper = Wire(new XSBundleWithMicroOp)
306e4f69d78Ssfencevma      wrapper.uop := uop
307e4f69d78Ssfencevma      wrapper
308e4f69d78Ssfencevma    })
309e4f69d78Ssfencevma
310e4f69d78Ssfencevma    // select logic
311520ec528SXuan Hu    val lqSelect: (Seq[Bool], Seq[XSBundleWithMicroOp]) = selectOldest(lqViolationSelVec, lqViolationSelUopExts)
312e4f69d78Ssfencevma
313e4f69d78Ssfencevma    // select one inst
314e4f69d78Ssfencevma    val lqViolation = lqSelect._1(0)
315e4f69d78Ssfencevma    val lqViolationUop = lqSelect._2(0).uop
316e4f69d78Ssfencevma
317e4f69d78Ssfencevma    XSDebug(
318e4f69d78Ssfencevma      lqViolation,
319e4f69d78Ssfencevma      "need rollback (ld wb before store) pc %x robidx %d target %x\n",
32020a5248fSzhanglinjuan      storeIn(i).bits.uop.pc, storeIn(i).bits.uop.robIdx.asUInt, lqViolationUop.robIdx.asUInt
321e4f69d78Ssfencevma    )
322e4f69d78Ssfencevma
323e4f69d78Ssfencevma    (lqViolation, lqViolationUop)
324e4f69d78Ssfencevma  }
325e4f69d78Ssfencevma
326e4f69d78Ssfencevma  // select rollback (part1) and generate rollback request
327e4f69d78Ssfencevma  // rollback check
328e4f69d78Ssfencevma  // Lq rollback seq check is done in s3 (next stage), as getting rollbackLq MicroOp is slow
32971489510SXuan Hu  val rollbackLqWb = Wire(Vec(StorePipelineWidth, Valid(new DynInst)))
330e4f69d78Ssfencevma  val stFtqIdx = Wire(Vec(StorePipelineWidth, new FtqPtr))
331e4f69d78Ssfencevma  val stFtqOffset = Wire(Vec(StorePipelineWidth, UInt(log2Up(PredictWidth).W)))
332e4f69d78Ssfencevma  for (w <- 0 until StorePipelineWidth) {
333e4f69d78Ssfencevma    val detectedRollback = detectRollback(w)
33420a5248fSzhanglinjuan    rollbackLqWb(w).valid := detectedRollback._1 && DelayN(storeIn(w).valid && !storeIn(w).bits.miss, TotalSelectCycles)
335cd2ff98bShappy-lx    rollbackLqWb(w).bits  := detectedRollback._2
3365003e6f8SHuijin Li    stFtqIdx(w) := DelayNWithValid(storeIn(w).bits.uop.ftqPtr, storeIn(w).valid, TotalSelectCycles)._2
3375003e6f8SHuijin Li    stFtqOffset(w) := DelayNWithValid(storeIn(w).bits.uop.ftqOffset, storeIn(w).valid, TotalSelectCycles)._2
338e4f69d78Ssfencevma  }
339e4f69d78Ssfencevma
340e4f69d78Ssfencevma  // select rollback (part2), generate rollback request, then fire rollback request
341e4f69d78Ssfencevma  // Note that we use robIdx - 1.U to flush the load instruction itself.
342e4f69d78Ssfencevma  // Thus, here if last cycle's robIdx equals to this cycle's robIdx, it still triggers the redirect.
343e4f69d78Ssfencevma
344e4f69d78Ssfencevma  // select uop in parallel
34516ede6bbSweiding liu
346cd2ff98bShappy-lx  val allRedirect = (0 until StorePipelineWidth).map(i => {
347cd2ff98bShappy-lx    val redirect = Wire(Valid(new Redirect))
348cd2ff98bShappy-lx    redirect.valid := rollbackLqWb(i).valid
349cd2ff98bShappy-lx    redirect.bits             := DontCare
35071489510SXuan Hu    redirect.bits.isRVC       := rollbackLqWb(i).bits.preDecodeInfo.isRVC
351cd2ff98bShappy-lx    redirect.bits.robIdx      := rollbackLqWb(i).bits.robIdx
3528241cb85SXuan Hu    redirect.bits.ftqIdx      := rollbackLqWb(i).bits.ftqPtr
3538241cb85SXuan Hu    redirect.bits.ftqOffset   := rollbackLqWb(i).bits.ftqOffset
354cd2ff98bShappy-lx    redirect.bits.stFtqIdx    := stFtqIdx(i)
355cd2ff98bShappy-lx    redirect.bits.stFtqOffset := stFtqOffset(i)
356cd2ff98bShappy-lx    redirect.bits.level       := RedirectLevel.flush
3578241cb85SXuan Hu    redirect.bits.cfiUpdate.target := rollbackLqWb(i).bits.pc
358cd2ff98bShappy-lx    redirect.bits.debug_runahead_checkpoint_id := rollbackLqWb(i).bits.debugInfo.runahead_checkpoint_id
359cd2ff98bShappy-lx    redirect
360cd2ff98bShappy-lx  })
36116ede6bbSweiding liu  io.rollback := allRedirect
362e4f69d78Ssfencevma
363e4f69d78Ssfencevma  // perf cnt
364e4f69d78Ssfencevma  val canEnqCount = PopCount(io.query.map(_.req.fire))
365e4f69d78Ssfencevma  val validCount = freeList.io.validCount
366e4f69d78Ssfencevma  val allowEnqueue = validCount <= (LoadQueueRAWSize - LoadPipelineWidth).U
36716ede6bbSweiding liu  val rollbaclValid = io.rollback.map(_.valid).reduce(_ || _).asUInt
368e4f69d78Ssfencevma
369e4f69d78Ssfencevma  QueuePerf(LoadQueueRAWSize, validCount, !allowEnqueue)
370e4f69d78Ssfencevma  XSPerfAccumulate("enqs", canEnqCount)
37116ede6bbSweiding liu  XSPerfAccumulate("stld_rollback", rollbaclValid)
372e4f69d78Ssfencevma  val perfEvents: Seq[(String, UInt)] = Seq(
373e4f69d78Ssfencevma    ("enq ", canEnqCount),
37416ede6bbSweiding liu    ("stld_rollback", rollbaclValid),
375e4f69d78Ssfencevma  )
376e4f69d78Ssfencevma  generatePerfEvent()
377e4f69d78Ssfencevma  // end
378e4f69d78Ssfencevma}
379