xref: /XiangShan/src/main/scala/xiangshan/backend/issue/IssueQueue.scala (revision a01a12bb633ce1ddf88408e652e979eaeb21aa57) 
1package xiangshan.backend.issue
2
3import org.chipsalliance.cde.config.Parameters
4import chisel3._
5import chisel3.util._
6import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp}
7import utility.{GTimer, HasCircularQueuePtrHelper, SelectOne}
8import utils._
9import xiangshan._
10import xiangshan.backend.Bundles._
11import xiangshan.backend.issue.EntryBundles._
12import xiangshan.backend.decode.{ImmUnion, Imm_LUI_LOAD}
13import xiangshan.backend.datapath.DataConfig._
14import xiangshan.backend.datapath.DataSource
15import xiangshan.backend.fu.{FuConfig, FuType}
16import xiangshan.mem.{MemWaitUpdateReq, SqPtr, LqPtr}
17import xiangshan.backend.rob.RobPtr
18import xiangshan.backend.datapath.NewPipelineConnect
19
20class IssueQueue(params: IssueBlockParams)(implicit p: Parameters) extends LazyModule with HasXSParameter {
21  override def shouldBeInlined: Boolean = false
22
23  implicit val iqParams = params
24  lazy val module: IssueQueueImp = iqParams.schdType match {
25    case IntScheduler() => new IssueQueueIntImp(this)
26    case VfScheduler() => new IssueQueueVfImp(this)
27    case MemScheduler() =>
28      if (iqParams.StdCnt == 0 && !iqParams.isVecMemIQ) new IssueQueueMemAddrImp(this)
29      else if (iqParams.isVecMemIQ) new IssueQueueVecMemImp(this)
30      else new IssueQueueIntImp(this)
31    case _ => null
32  }
33}
34
35class IssueQueueStatusBundle(numEnq: Int, numEntries: Int) extends Bundle {
36  val empty = Output(Bool())
37  val full = Output(Bool())
38  val validCnt = Output(UInt(log2Ceil(numEntries).W))
39  val leftVec = Output(Vec(numEnq + 1, Bool()))
40}
41
42class IssueQueueDeqRespBundle(implicit p:Parameters, params: IssueBlockParams) extends EntryDeqRespBundle
43
44class IssueQueueIO()(implicit p: Parameters, params: IssueBlockParams) extends XSBundle {
45  // Inputs
46  val flush = Flipped(ValidIO(new Redirect))
47  val enq = Vec(params.numEnq, Flipped(DecoupledIO(new DynInst)))
48
49  val og0Resp = Vec(params.numDeq, Flipped(ValidIO(new IssueQueueDeqRespBundle)))
50  val og1Resp = Vec(params.numDeq, Flipped(ValidIO(new IssueQueueDeqRespBundle)))
51  val finalIssueResp = OptionWrapper(params.LdExuCnt > 0, Vec(params.numDeq, Flipped(ValidIO(new IssueQueueDeqRespBundle))))
52  val memAddrIssueResp = OptionWrapper(params.LdExuCnt > 0, Vec(params.numDeq, Flipped(ValidIO(new IssueQueueDeqRespBundle))))
53  val wbBusyTableRead = Input(params.genWbFuBusyTableReadBundle())
54  val wbBusyTableWrite = Output(params.genWbFuBusyTableWriteBundle())
55  val wakeupFromWB: MixedVec[ValidIO[IssueQueueWBWakeUpBundle]] = Flipped(params.genWBWakeUpSinkValidBundle)
56  val wakeupFromIQ: MixedVec[ValidIO[IssueQueueIQWakeUpBundle]] = Flipped(params.genIQWakeUpSinkValidBundle)
57  val og0Cancel = Input(ExuOH(backendParams.numExu))
58  val og1Cancel = Input(ExuOH(backendParams.numExu))
59  val ldCancel = Vec(backendParams.LduCnt + backendParams.HyuCnt, Flipped(new LoadCancelIO))
60
61  // Outputs
62  val wakeupToIQ: MixedVec[ValidIO[IssueQueueIQWakeUpBundle]] = params.genIQWakeUpSourceValidBundle
63  val status = Output(new IssueQueueStatusBundle(params.numEnq, params.numEntries))
64  // val statusNext = Output(new IssueQueueStatusBundle(params.numEnq))
65
66  val deqDelay: MixedVec[DecoupledIO[IssueQueueIssueBundle]] = params.genIssueDecoupledBundle// = deq.cloneType
67  def allWakeUp = wakeupFromWB ++ wakeupFromIQ
68}
69
70class IssueQueueImp(override val wrapper: IssueQueue)(implicit p: Parameters, val params: IssueBlockParams)
71  extends LazyModuleImp(wrapper)
72  with HasXSParameter {
73
74  override def desiredName: String = s"${params.getIQName}"
75
76  println(s"[IssueQueueImp] ${params.getIQName} wakeupFromWB(${io.wakeupFromWB.size}), " +
77    s"wakeup exu in(${params.wakeUpInExuSources.size}): ${params.wakeUpInExuSources.map(_.name).mkString("{",",","}")}, " +
78    s"wakeup exu out(${params.wakeUpOutExuSources.size}): ${params.wakeUpOutExuSources.map(_.name).mkString("{",",","}")}, " +
79    s"numEntries: ${params.numEntries}, numRegSrc: ${params.numRegSrc}")
80
81  require(params.numExu <= 2, "IssueQueue has not supported more than 2 deq ports")
82  val deqFuCfgs     : Seq[Seq[FuConfig]] = params.exuBlockParams.map(_.fuConfigs)
83  val allDeqFuCfgs  : Seq[FuConfig] = params.exuBlockParams.flatMap(_.fuConfigs)
84  val fuCfgsCnt     : Map[FuConfig, Int] = allDeqFuCfgs.groupBy(x => x).map { case (cfg, cfgSeq) => (cfg, cfgSeq.length) }
85  val commonFuCfgs  : Seq[FuConfig] = fuCfgsCnt.filter(_._2 > 1).keys.toSeq
86  val fuLatencyMaps : Seq[Map[FuType.OHType, Int]] = params.exuBlockParams.map(x => x.fuLatencyMap)
87
88  println(s"[IssueQueueImp] ${params.getIQName} fuLatencyMaps: ${fuLatencyMaps}")
89  println(s"[IssueQueueImp] ${params.getIQName} commonFuCfgs: ${commonFuCfgs.map(_.name)}")
90  lazy val io = IO(new IssueQueueIO())
91  // Modules
92
93  val entries = Module(new Entries)
94  val fuBusyTableWrite = params.exuBlockParams.map { case x => OptionWrapper(x.latencyValMax > 0, Module(new FuBusyTableWrite(x.fuLatencyMap))) }
95  val fuBusyTableRead = params.exuBlockParams.map { case x => OptionWrapper(x.latencyValMax > 0, Module(new FuBusyTableRead(x.fuLatencyMap))) }
96  val intWbBusyTableWrite = params.exuBlockParams.map { case x => OptionWrapper(x.intLatencyCertain, Module(new FuBusyTableWrite(x.intFuLatencyMap))) }
97  val intWbBusyTableRead = params.exuBlockParams.map { case x => OptionWrapper(x.intLatencyCertain, Module(new FuBusyTableRead(x.intFuLatencyMap))) }
98  val vfWbBusyTableWrite = params.exuBlockParams.map { case x => OptionWrapper(x.vfLatencyCertain, Module(new FuBusyTableWrite(x.vfFuLatencyMap))) }
99  val vfWbBusyTableRead = params.exuBlockParams.map { case x => OptionWrapper(x.vfLatencyCertain, Module(new FuBusyTableRead(x.vfFuLatencyMap))) }
100
101  class WakeupQueueFlush extends Bundle {
102    val redirect = ValidIO(new Redirect)
103    val ldCancel = Vec(backendParams.LduCnt + backendParams.HyuCnt, new LoadCancelIO)
104    val og0Fail = Output(Bool())
105    val og1Fail = Output(Bool())
106  }
107
108  private def flushFunc(exuInput: ExuInput, flush: WakeupQueueFlush, stage: Int): Bool = {
109    val redirectFlush = exuInput.robIdx.needFlush(flush.redirect)
110    val loadDependencyFlush = LoadShouldCancel(exuInput.loadDependency, flush.ldCancel)
111    val ogFailFlush = stage match {
112      case 1 => flush.og0Fail
113      case 2 => flush.og1Fail
114      case _ => false.B
115    }
116    redirectFlush || loadDependencyFlush || ogFailFlush
117  }
118
119  private def modificationFunc(exuInput: ExuInput): ExuInput = {
120    val newExuInput = WireDefault(exuInput)
121    newExuInput.loadDependency match {
122      case Some(deps) => deps.zip(exuInput.loadDependency.get).foreach(x => x._1 := x._2 << 1)
123      case None =>
124    }
125    newExuInput
126  }
127
128  private def lastConnectFunc(exuInput: ExuInput, newInput: ExuInput): ExuInput = {
129    val lastExuInput = WireDefault(exuInput)
130    val newExuInput = WireDefault(newInput)
131    newExuInput.elements.foreach { case (name, data) =>
132      if (lastExuInput.elements.contains(name)) {
133        data := lastExuInput.elements(name)
134      }
135    }
136    if (newExuInput.pdestCopy.nonEmpty && !lastExuInput.pdestCopy.nonEmpty) {
137      newExuInput.pdestCopy.get.foreach(_ := lastExuInput.pdest)
138    }
139    if (newExuInput.rfWenCopy.nonEmpty && !lastExuInput.rfWenCopy.nonEmpty) {
140      newExuInput.rfWenCopy.get.foreach(_ := lastExuInput.rfWen.get)
141    }
142    if (newExuInput.fpWenCopy.nonEmpty && !lastExuInput.fpWenCopy.nonEmpty) {
143      newExuInput.fpWenCopy.get.foreach(_ := lastExuInput.fpWen.get)
144    }
145    if (newExuInput.vecWenCopy.nonEmpty && !lastExuInput.vecWenCopy.nonEmpty) {
146      newExuInput.vecWenCopy.get.foreach(_ := lastExuInput.rfWen.get)
147    }
148    if (newExuInput.loadDependencyCopy.nonEmpty && !lastExuInput.loadDependencyCopy.nonEmpty) {
149      newExuInput.loadDependencyCopy.get.foreach(_ := lastExuInput.loadDependency.get)
150    }
151    newExuInput
152  }
153
154  val wakeUpQueues: Seq[Option[MultiWakeupQueue[ExuInput, WakeupQueueFlush]]] = params.exuBlockParams.map { x => OptionWrapper(x.isIQWakeUpSource && !x.hasLoadExu, Module(
155    new MultiWakeupQueue(new ExuInput(x), new ExuInput(x, x.copyWakeupOut, x.copyNum), new WakeupQueueFlush, x.fuLatancySet, flushFunc, modificationFunc, lastConnectFunc)
156  ))}
157  val deqBeforeDly = Wire(params.genIssueDecoupledBundle)
158
159  val intWbBusyTableIn = io.wbBusyTableRead.map(_.intWbBusyTable)
160  val vfWbBusyTableIn = io.wbBusyTableRead.map(_.vfWbBusyTable)
161  val intWbBusyTableOut = io.wbBusyTableWrite.map(_.intWbBusyTable)
162  val vfWbBusyTableOut = io.wbBusyTableWrite.map(_.vfWbBusyTable)
163  val intDeqRespSetOut = io.wbBusyTableWrite.map(_.intDeqRespSet)
164  val vfDeqRespSetOut = io.wbBusyTableWrite.map(_.vfDeqRespSet)
165  val fuBusyTableMask = Wire(Vec(params.numDeq, UInt(params.numEntries.W)))
166  val intWbBusyTableMask = Wire(Vec(params.numDeq, UInt(params.numEntries.W)))
167  val vfWbBusyTableMask = Wire(Vec(params.numDeq, UInt(params.numEntries.W)))
168  val s0_enqValidVec = io.enq.map(_.valid)
169  val s0_enqSelValidVec = Wire(Vec(params.numEnq, Bool()))
170  val s0_enqNotFlush = !io.flush.valid
171  val s0_enqBits = WireInit(VecInit(io.enq.map(_.bits)))
172  val s0_doEnqSelValidVec = s0_enqSelValidVec.map(_ && s0_enqNotFlush) //enqValid && notFlush && enqReady
173
174
175  val finalDeqSelValidVec = Wire(Vec(params.numDeq, Bool()))
176  val finalDeqSelOHVec    = Wire(Vec(params.numDeq, UInt(params.numEntries.W)))
177
178  val validVec = VecInit(entries.io.valid.asBools)
179  val canIssueVec = VecInit(entries.io.canIssue.asBools)
180  dontTouch(canIssueVec)
181  val deqFirstIssueVec = entries.io.isFirstIssue
182
183  val dataSources: Vec[Vec[DataSource]] = entries.io.dataSources
184  val finalDataSources: Vec[Vec[DataSource]] = VecInit(finalDeqSelOHVec.map(oh => Mux1H(oh, dataSources)))
185  // (entryIdx)(srcIdx)(exuIdx)
186  val wakeUpL1ExuOH: Option[Vec[Vec[UInt]]] = entries.io.srcWakeUpL1ExuOH
187  val srcTimer: Option[Vec[Vec[UInt]]] = entries.io.srcTimer
188
189  // (deqIdx)(srcIdx)(exuIdx)
190  val finalWakeUpL1ExuOH: Option[Vec[Vec[UInt]]] = wakeUpL1ExuOH.map(x => VecInit(finalDeqSelOHVec.map(oh => Mux1H(oh, x))))
191  val finalSrcTimer = srcTimer.map(x => VecInit(finalDeqSelOHVec.map(oh => Mux1H(oh, x))))
192
193  val fuTypeVec = Wire(Vec(params.numEntries, FuType()))
194  val transEntryDeqVec = Wire(Vec(params.numEnq, ValidIO(new EntryBundle)))
195  val deqEntryVec = Wire(Vec(params.numDeq, ValidIO(new EntryBundle)))
196  val transSelVec = Wire(Vec(params.numEnq, UInt((params.numEntries-params.numEnq).W)))
197  val canIssueMergeAllBusy = Wire(Vec(params.numDeq, UInt(params.numEntries.W)))
198  val deqCanIssue = Wire(Vec(params.numDeq, UInt(params.numEntries.W)))
199
200  val enqEntryOldestSel = Wire(Vec(params.numDeq, ValidIO(UInt(params.numEnq.W))))
201  val othersEntryOldestSel = Wire(Vec(params.numDeq, ValidIO(UInt((params.numEntries - params.numEnq).W))))
202  val deqSelValidVec = Wire(Vec(params.numDeq, Bool()))
203  val deqSelOHVec    = Wire(Vec(params.numDeq, UInt(params.numEntries.W)))
204  val cancelDeqVec = Wire(Vec(params.numDeq, Bool()))
205
206  val subDeqSelValidVec = OptionWrapper(params.deqFuSame, Wire(Vec(params.numDeq, Bool())))
207  val subDeqSelOHVec = OptionWrapper(params.deqFuSame, Wire(Vec(params.numDeq, UInt(params.numEntries.W))))
208  val subDeqRequest = OptionWrapper(params.deqFuSame, Wire(UInt(params.numEntries.W)))
209
210  /**
211    * Connection of [[entries]]
212    */
213  entries.io match { case entriesIO: EntriesIO =>
214    entriesIO.flush                                             := io.flush
215    entriesIO.enq.zipWithIndex.foreach { case (enq, enqIdx) =>
216      enq.valid                                                 := s0_doEnqSelValidVec(enqIdx)
217      enq.bits.status.robIdx                                    := s0_enqBits(enqIdx).robIdx
218      enq.bits.status.fuType                                    := IQFuType.readFuType(VecInit(s0_enqBits(enqIdx).fuType.asBools), params.getFuCfgs.map(_.fuType))
219      val numLsrc = s0_enqBits(enqIdx).srcType.size.min(enq.bits.status.srcStatus.map(_.srcType).size)
220      for(j <- 0 until numLsrc) {
221        enq.bits.status.srcStatus(j).psrc                       := s0_enqBits(enqIdx).psrc(j)
222        enq.bits.status.srcStatus(j).srcType                    := s0_enqBits(enqIdx).srcType(j)
223        enq.bits.status.srcStatus(j).srcState                   := s0_enqBits(enqIdx).srcState(j) & !LoadShouldCancel(Some(s0_enqBits(enqIdx).srcLoadDependency(j)), io.ldCancel)
224        enq.bits.status.srcStatus(j).dataSources.value          := DataSource.reg
225        if(params.hasIQWakeUp) {
226          enq.bits.status.srcStatus(j).srcTimer.get             := 0.U(3.W)
227          enq.bits.status.srcStatus(j).srcWakeUpL1ExuOH.get     := 0.U.asTypeOf(ExuVec())
228          enq.bits.status.srcStatus(j).srcLoadDependency.get    := VecInit(s0_enqBits(enqIdx).srcLoadDependency(j).map(x => x(x.getWidth - 2, 0) << 1))
229        }
230      }
231      enq.bits.status.blocked                                   := false.B
232      enq.bits.status.issued                                    := false.B
233      enq.bits.status.firstIssue                                := false.B
234      enq.bits.status.issueTimer                                := "b10".U
235      enq.bits.status.deqPortIdx                                := 0.U
236      if (params.isVecMemIQ) {
237        enq.bits.status.vecMem.get.uopIdx := s0_enqBits(enqIdx).uopIdx
238      }
239      if (params.inIntSchd && params.AluCnt > 0) {
240        // dirty code for lui+addi(w) fusion
241        val isLuiAddiFusion = s0_enqBits(enqIdx).isLUI32
242        val luiImm = Cat(s0_enqBits(enqIdx).lsrc(1), s0_enqBits(enqIdx).lsrc(0), s0_enqBits(enqIdx).imm(ImmUnion.maxLen - 1, 0))
243        enq.bits.imm.foreach(_ := Mux(isLuiAddiFusion, ImmUnion.LUI32.toImm32(luiImm), s0_enqBits(enqIdx).imm))
244      }
245      else if (params.inMemSchd && params.LduCnt > 0) {
246        // dirty code for fused_lui_load
247        val isLuiLoadFusion = SrcType.isNotReg(s0_enqBits(enqIdx).srcType(0)) && FuType.isLoad(s0_enqBits(enqIdx).fuType)
248        enq.bits.imm.foreach(_ := Mux(isLuiLoadFusion, Imm_LUI_LOAD().getLuiImm(s0_enqBits(enqIdx)), s0_enqBits(enqIdx).imm))
249      }
250      else {
251        enq.bits.imm.foreach(_ := s0_enqBits(enqIdx).imm)
252      }
253      enq.bits.payload                                          := s0_enqBits(enqIdx)
254    }
255    entriesIO.og0Resp.zipWithIndex.foreach { case (og0Resp, i) =>
256      og0Resp.valid                                             := io.og0Resp(i).valid
257      og0Resp.bits.robIdx                                       := io.og0Resp(i).bits.robIdx
258      og0Resp.bits.uopIdx.foreach(_                             := io.og0Resp(i).bits.uopIdx.get)
259      og0Resp.bits.dataInvalidSqIdx                             := io.og0Resp(i).bits.dataInvalidSqIdx
260      og0Resp.bits.respType                                     := io.og0Resp(i).bits.respType
261      og0Resp.bits.rfWen                                        := io.og0Resp(i).bits.rfWen
262      og0Resp.bits.fuType                                       := io.og0Resp(i).bits.fuType
263    }
264    entriesIO.og1Resp.zipWithIndex.foreach { case (og1Resp, i) =>
265      og1Resp.valid                                             := io.og1Resp(i).valid
266      og1Resp.bits.robIdx                                       := io.og1Resp(i).bits.robIdx
267      og1Resp.bits.uopIdx.foreach(_                             := io.og1Resp(i).bits.uopIdx.get)
268      og1Resp.bits.dataInvalidSqIdx                             := io.og1Resp(i).bits.dataInvalidSqIdx
269      og1Resp.bits.respType                                     := io.og1Resp(i).bits.respType
270      og1Resp.bits.rfWen                                        := io.og1Resp(i).bits.rfWen
271      og1Resp.bits.fuType                                       := io.og1Resp(i).bits.fuType
272    }
273    entriesIO.finalIssueResp.foreach(_.zipWithIndex.foreach { case (finalIssueResp, i) =>
274      finalIssueResp                                            := io.finalIssueResp.get(i)
275    })
276    for(deqIdx <- 0 until params.numDeq) {
277      entriesIO.deqReady(deqIdx)                                := deqBeforeDly(deqIdx).ready
278      entriesIO.deqSelOH(deqIdx).valid                          := deqSelValidVec(deqIdx)
279      entriesIO.deqSelOH(deqIdx).bits                           := deqSelOHVec(deqIdx)
280      entriesIO.enqEntryOldestSel(deqIdx)                       := enqEntryOldestSel(deqIdx)
281      entriesIO.othersEntryOldestSel(deqIdx)                    := othersEntryOldestSel(deqIdx)
282      entriesIO.subDeqRequest.foreach(_(deqIdx)                 := subDeqRequest.get)
283      entriesIO.subDeqSelOH.foreach(_(deqIdx)                   := subDeqSelOHVec.get(deqIdx))
284    }
285    entriesIO.wakeUpFromWB                                      := io.wakeupFromWB
286    entriesIO.wakeUpFromIQ                                      := io.wakeupFromIQ
287    entriesIO.og0Cancel                                         := io.og0Cancel
288    entriesIO.og1Cancel                                         := io.og1Cancel
289    entriesIO.ldCancel                                          := io.ldCancel
290    //output
291    transEntryDeqVec                                            := entriesIO.transEntryDeqVec
292    transSelVec                                                 := entriesIO.transSelVec
293    fuTypeVec                                                   := entriesIO.fuType
294    deqEntryVec                                                 := entriesIO.deqEntry
295    cancelDeqVec                                                := entriesIO.cancelDeqVec
296  }
297
298
299  s0_enqSelValidVec := s0_enqValidVec.zip(io.enq).map{ case (enqValid, enq) => enqValid && enq.ready}
300
301  protected val commonAccept: UInt = Cat(fuTypeVec.map(fuType =>
302    FuType.FuTypeOrR(fuType, commonFuCfgs.map(_.fuType))
303  ).reverse)
304
305  // if deq port can accept the uop
306  protected val canAcceptVec: Seq[UInt] = deqFuCfgs.map { fuCfgs: Seq[FuConfig] =>
307    Cat(fuTypeVec.map(fuType =>
308      FuType.FuTypeOrR(fuType, fuCfgs.map(_.fuType))
309    ).reverse)
310  }
311
312  protected val deqCanAcceptVec: Seq[IndexedSeq[Bool]] = deqFuCfgs.map { fuCfgs: Seq[FuConfig] =>
313    fuTypeVec.map(fuType =>
314      FuType.FuTypeOrR(fuType, fuCfgs.map(_.fuType)))
315  }
316
317  canIssueMergeAllBusy.zipWithIndex.foreach { case (merge, i) =>
318    val mergeFuBusy = {
319      if (fuBusyTableWrite(i).nonEmpty) canIssueVec.asUInt & (~fuBusyTableMask(i))
320      else canIssueVec.asUInt
321    }
322    val mergeIntWbBusy = {
323      if (intWbBusyTableRead(i).nonEmpty) mergeFuBusy & (~intWbBusyTableMask(i))
324      else mergeFuBusy
325    }
326    val mergeVfWbBusy = {
327      if (vfWbBusyTableRead(i).nonEmpty) mergeIntWbBusy & (~vfWbBusyTableMask(i))
328      else mergeIntWbBusy
329    }
330    merge := mergeVfWbBusy
331  }
332
333  deqCanIssue.zipWithIndex.foreach { case (req, i) =>
334    req := canIssueMergeAllBusy(i) & VecInit(deqCanAcceptVec(i)).asUInt
335  }
336  dontTouch(fuTypeVec)
337  dontTouch(canIssueMergeAllBusy)
338  dontTouch(deqCanIssue)
339
340  if (params.numDeq == 2) {
341    require(params.deqFuSame || params.deqFuDiff, "The 2 deq ports need to be identical or completely different")
342  }
343
344  if (params.numDeq == 2 && params.deqFuSame) {
345    enqEntryOldestSel := DontCare
346
347    othersEntryOldestSel(0) := AgeDetector(numEntries = params.numEntries - params.numEnq,
348      enq = VecInit(transEntryDeqVec.zip(transSelVec).map{ case (transEntry, transSel) => Fill(params.numEntries-params.numEnq, transEntry.valid) & transSel }),
349      canIssue = canIssueVec.asUInt(params.numEntries-1, params.numEnq)
350    )
351    othersEntryOldestSel(1) := DontCare
352
353    subDeqRequest.get := canIssueVec.asUInt & ~Cat(othersEntryOldestSel(0).bits, 0.U((params.numEnq).W))
354
355    val subDeqPolicy = Module(new DeqPolicy())
356    subDeqPolicy.io.request := subDeqRequest.get
357    subDeqSelValidVec.get := subDeqPolicy.io.deqSelOHVec.map(oh => oh.valid)
358    subDeqSelOHVec.get := subDeqPolicy.io.deqSelOHVec.map(oh => oh.bits)
359
360    deqSelValidVec(0) := othersEntryOldestSel(0).valid || subDeqSelValidVec.get(1)
361    deqSelValidVec(1) := subDeqSelValidVec.get(0)
362    deqSelOHVec(0) := Mux(othersEntryOldestSel(0).valid,
363                          Cat(othersEntryOldestSel(0).bits, 0.U((params.numEnq).W)),
364                          subDeqSelOHVec.get(1)) & canIssueMergeAllBusy(0)
365    deqSelOHVec(1) := subDeqSelOHVec.get(0) & canIssueMergeAllBusy(1)
366
367    finalDeqSelValidVec.zip(finalDeqSelOHVec).zip(deqSelValidVec).zip(deqSelOHVec).zipWithIndex.foreach { case ((((selValid, selOH), deqValid), deqOH), i) =>
368      selValid := deqValid && deqOH.orR && deqBeforeDly(i).ready
369      selOH := deqOH
370    }
371  }
372  else {
373    enqEntryOldestSel := NewAgeDetector(numEntries = params.numEnq,
374      enq = VecInit(s0_doEnqSelValidVec),
375      canIssue = VecInit(deqCanIssue.map(_(params.numEnq-1, 0)))
376    )
377
378    othersEntryOldestSel := AgeDetector(numEntries = params.numEntries - params.numEnq,
379      enq = VecInit(transEntryDeqVec.zip(transSelVec).map{ case (transEntry, transSel) => Fill(params.numEntries-params.numEnq, transEntry.valid) & transSel }),
380      canIssue = VecInit(deqCanIssue.map(_(params.numEntries-1, params.numEnq)))
381    )
382
383    deqSelValidVec.zip(deqSelOHVec).zipWithIndex.foreach { case ((selValid, selOH), i) =>
384      if (params.exuBlockParams(i).fuConfigs.contains(FuConfig.FakeHystaCfg)) {
385        selValid := false.B
386        selOH := 0.U.asTypeOf(selOH)
387      } else {
388        selValid := othersEntryOldestSel(i).valid || enqEntryOldestSel(i).valid
389        selOH := Cat(othersEntryOldestSel(i).bits, Fill(params.numEnq, enqEntryOldestSel(i).valid && !othersEntryOldestSel(i).valid) & enqEntryOldestSel(i).bits)
390      }
391    }
392
393    finalDeqSelValidVec.zip(finalDeqSelOHVec).zip(deqSelValidVec).zip(deqSelOHVec).zipWithIndex.foreach { case ((((selValid, selOH), deqValid), deqOH), i) =>
394      selValid := deqValid && deqBeforeDly(i).ready
395      selOH := deqOH
396    }
397  }
398
399  val toBusyTableDeqResp = Wire(Vec(params.numDeq, ValidIO(new IssueQueueDeqRespBundle)))
400
401  toBusyTableDeqResp.zipWithIndex.foreach { case (deqResp, i) =>
402    deqResp.valid := finalDeqSelValidVec(i)
403    deqResp.bits.respType := RSFeedbackType.issueSuccess
404    deqResp.bits.robIdx := DontCare
405    deqResp.bits.dataInvalidSqIdx := DontCare
406    deqResp.bits.rfWen := DontCare
407    deqResp.bits.fuType := deqBeforeDly(i).bits.common.fuType
408    deqResp.bits.uopIdx.foreach(_ := DontCare)
409  }
410
411  //fuBusyTable
412  fuBusyTableWrite.zip(fuBusyTableRead).zipWithIndex.foreach { case ((busyTableWrite: Option[FuBusyTableWrite], busyTableRead: Option[FuBusyTableRead]), i) =>
413    if(busyTableWrite.nonEmpty) {
414      val btwr = busyTableWrite.get
415      val btrd = busyTableRead.get
416      btwr.io.in.deqResp := toBusyTableDeqResp(i)
417      btwr.io.in.og0Resp := io.og0Resp(i)
418      btwr.io.in.og1Resp := io.og1Resp(i)
419      btrd.io.in.fuBusyTable := btwr.io.out.fuBusyTable
420      btrd.io.in.fuTypeRegVec := fuTypeVec
421      fuBusyTableMask(i) := btrd.io.out.fuBusyTableMask
422    }
423    else {
424      fuBusyTableMask(i) := 0.U(params.numEntries.W)
425    }
426  }
427
428  //wbfuBusyTable write
429  intWbBusyTableWrite.zip(intWbBusyTableOut).zip(intDeqRespSetOut).zipWithIndex.foreach { case (((busyTableWrite: Option[FuBusyTableWrite], busyTable: Option[UInt]), deqResp), i) =>
430    if(busyTableWrite.nonEmpty) {
431      val btwr = busyTableWrite.get
432      val bt = busyTable.get
433      val dq = deqResp.get
434      btwr.io.in.deqResp := toBusyTableDeqResp(i)
435      btwr.io.in.og0Resp := io.og0Resp(i)
436      btwr.io.in.og1Resp := io.og1Resp(i)
437      bt := btwr.io.out.fuBusyTable
438      dq := btwr.io.out.deqRespSet
439    }
440  }
441
442  vfWbBusyTableWrite.zip(vfWbBusyTableOut).zip(vfDeqRespSetOut).zipWithIndex.foreach { case (((busyTableWrite: Option[FuBusyTableWrite], busyTable: Option[UInt]), deqResp), i) =>
443    if (busyTableWrite.nonEmpty) {
444      val btwr = busyTableWrite.get
445      val bt = busyTable.get
446      val dq = deqResp.get
447      btwr.io.in.deqResp := toBusyTableDeqResp(i)
448      btwr.io.in.og0Resp := io.og0Resp(i)
449      btwr.io.in.og1Resp := io.og1Resp(i)
450      bt := btwr.io.out.fuBusyTable
451      dq := btwr.io.out.deqRespSet
452    }
453  }
454
455  //wbfuBusyTable read
456  intWbBusyTableRead.zip(intWbBusyTableIn).zipWithIndex.foreach { case ((busyTableRead: Option[FuBusyTableRead], busyTable: Option[UInt]), i) =>
457    if(busyTableRead.nonEmpty) {
458      val btrd = busyTableRead.get
459      val bt = busyTable.get
460      btrd.io.in.fuBusyTable := bt
461      btrd.io.in.fuTypeRegVec := fuTypeVec
462      intWbBusyTableMask(i) := btrd.io.out.fuBusyTableMask
463    }
464    else {
465      intWbBusyTableMask(i) := 0.U(params.numEntries.W)
466    }
467  }
468  vfWbBusyTableRead.zip(vfWbBusyTableIn).zipWithIndex.foreach { case ((busyTableRead: Option[FuBusyTableRead], busyTable: Option[UInt]), i) =>
469    if (busyTableRead.nonEmpty) {
470      val btrd = busyTableRead.get
471      val bt = busyTable.get
472      btrd.io.in.fuBusyTable := bt
473      btrd.io.in.fuTypeRegVec := fuTypeVec
474      vfWbBusyTableMask(i) := btrd.io.out.fuBusyTableMask
475    }
476    else {
477      vfWbBusyTableMask(i) := 0.U(params.numEntries.W)
478    }
479  }
480
481  wakeUpQueues.zipWithIndex.foreach { case (wakeUpQueueOption, i) =>
482    wakeUpQueueOption.foreach {
483      wakeUpQueue =>
484        val flush = Wire(new WakeupQueueFlush)
485        flush.redirect := io.flush
486        flush.ldCancel := io.ldCancel
487        flush.og0Fail := io.og0Resp(i).valid && RSFeedbackType.isBlocked(io.og0Resp(i).bits.respType)
488        flush.og1Fail := io.og1Resp(i).valid && RSFeedbackType.isBlocked(io.og1Resp(i).bits.respType)
489        wakeUpQueue.io.flush := flush
490        wakeUpQueue.io.enq.valid := deqBeforeDly(i).fire
491        wakeUpQueue.io.enq.bits.uop :<= deqBeforeDly(i).bits.common
492        wakeUpQueue.io.enq.bits.uop.pdestCopy.foreach(_ := 0.U)
493        wakeUpQueue.io.enq.bits.lat := getDeqLat(i, deqBeforeDly(i).bits.common.fuType)
494    }
495  }
496
497  deqBeforeDly.zipWithIndex.foreach { case (deq, i) =>
498    deq.valid                := finalDeqSelValidVec(i) && !cancelDeqVec(i)
499    deq.bits.addrOH          := finalDeqSelOHVec(i)
500    deq.bits.common.isFirstIssue := deqFirstIssueVec(i)
501    deq.bits.common.iqIdx    := OHToUInt(finalDeqSelOHVec(i))
502    deq.bits.common.fuType   := IQFuType.readFuType(deqEntryVec(i).bits.status.fuType, params.getFuCfgs.map(_.fuType)).asUInt
503    deq.bits.common.fuOpType := deqEntryVec(i).bits.payload.fuOpType
504    deq.bits.common.rfWen.foreach(_ := deqEntryVec(i).bits.payload.rfWen)
505    deq.bits.common.fpWen.foreach(_ := deqEntryVec(i).bits.payload.fpWen)
506    deq.bits.common.vecWen.foreach(_ := deqEntryVec(i).bits.payload.vecWen)
507    deq.bits.common.flushPipe.foreach(_ := deqEntryVec(i).bits.payload.flushPipe)
508    deq.bits.common.pdest := deqEntryVec(i).bits.payload.pdest
509    deq.bits.common.robIdx := deqEntryVec(i).bits.status.robIdx
510
511    require(deq.bits.common.dataSources.size <= finalDataSources(i).size)
512    deq.bits.common.dataSources.zip(finalDataSources(i)).foreach { case (sink, source) => sink := source}
513    deq.bits.common.l1ExuOH.foreach(_ := finalWakeUpL1ExuOH.get(i))
514    deq.bits.common.srcTimer.foreach(_ := finalSrcTimer.get(i))
515    deq.bits.common.loadDependency.foreach(_ := deqEntryVec(i).bits.status.mergedLoadDependency.get)
516    deq.bits.common.src := DontCare
517    deq.bits.common.preDecode.foreach(_ := deqEntryVec(i).bits.payload.preDecodeInfo)
518
519    deq.bits.rf.zip(deqEntryVec(i).bits.status.srcStatus.map(_.psrc)).zip(deqEntryVec(i).bits.status.srcStatus.map(_.srcType)).foreach { case ((rf, psrc), srcType) =>
520      // psrc in status array can be pregIdx of IntRegFile or VfRegFile
521      rf.foreach(_.addr := psrc)
522      rf.foreach(_.srcType := srcType)
523    }
524    deq.bits.srcType.zip(deqEntryVec(i).bits.status.srcStatus.map(_.srcType)).foreach { case (sink, source) =>
525      sink := source
526    }
527    deq.bits.immType := deqEntryVec(i).bits.payload.selImm
528    deq.bits.common.imm := deqEntryVec(i).bits.imm.getOrElse(0.U)
529
530    deq.bits.common.perfDebugInfo := deqEntryVec(i).bits.payload.debugInfo
531    deq.bits.common.perfDebugInfo.selectTime := GTimer()
532    deq.bits.common.perfDebugInfo.issueTime := GTimer() + 1.U
533  }
534
535  private val deqShift = WireDefault(deqBeforeDly)
536  deqShift.zip(deqBeforeDly).foreach {
537    case (shifted, original) =>
538      original.ready := shifted.ready // this will not cause combinational loop
539      shifted.bits.common.loadDependency.foreach(
540        _ := original.bits.common.loadDependency.get.map(_ << 1)
541      )
542  }
543  io.deqDelay.zip(deqShift).foreach { case (deqDly, deq) =>
544    NewPipelineConnect(
545      deq, deqDly, deqDly.valid,
546      false.B,
547      Option("Scheduler2DataPathPipe")
548    )
549  }
550  if(backendParams.debugEn) {
551    dontTouch(io.deqDelay)
552  }
553  io.wakeupToIQ.zipWithIndex.foreach { case (wakeup, i) =>
554    if (wakeUpQueues(i).nonEmpty && finalWakeUpL1ExuOH.nonEmpty) {
555      wakeup.valid := wakeUpQueues(i).get.io.deq.valid
556      wakeup.bits.fromExuInput(wakeUpQueues(i).get.io.deq.bits, finalWakeUpL1ExuOH.get(i))
557      wakeup.bits.loadDependency := wakeUpQueues(i).get.io.deq.bits.loadDependency.getOrElse(0.U.asTypeOf(wakeup.bits.loadDependency))
558      wakeup.bits.is0Lat := getDeqLat(i, wakeUpQueues(i).get.io.deq.bits.fuType) === 0.U
559    } else if (wakeUpQueues(i).nonEmpty) {
560      wakeup.valid := wakeUpQueues(i).get.io.deq.valid
561      wakeup.bits.fromExuInput(wakeUpQueues(i).get.io.deq.bits)
562      wakeup.bits.loadDependency := wakeUpQueues(i).get.io.deq.bits.loadDependency.getOrElse(0.U.asTypeOf(wakeup.bits.loadDependency))
563      wakeup.bits.is0Lat := getDeqLat(i, wakeUpQueues(i).get.io.deq.bits.fuType) === 0.U
564    } else {
565      wakeup.valid := false.B
566      wakeup.bits := 0.U.asTypeOf(wakeup.bits)
567      wakeup.bits.is0Lat :=  0.U
568    }
569    if (wakeUpQueues(i).nonEmpty) {
570      wakeup.bits.rfWen  := (if (wakeUpQueues(i).get.io.deq.bits.rfWen .nonEmpty) wakeUpQueues(i).get.io.deq.valid && wakeUpQueues(i).get.io.deq.bits.rfWen .get else false.B)
571      wakeup.bits.fpWen  := (if (wakeUpQueues(i).get.io.deq.bits.fpWen .nonEmpty) wakeUpQueues(i).get.io.deq.valid && wakeUpQueues(i).get.io.deq.bits.fpWen .get else false.B)
572      wakeup.bits.vecWen := (if (wakeUpQueues(i).get.io.deq.bits.vecWen.nonEmpty) wakeUpQueues(i).get.io.deq.valid && wakeUpQueues(i).get.io.deq.bits.vecWen.get else false.B)
573    }
574
575    if(wakeUpQueues(i).nonEmpty && wakeup.bits.pdestCopy.nonEmpty){
576      wakeup.bits.pdestCopy.get := wakeUpQueues(i).get.io.deq.bits.pdestCopy.get
577    }
578    if (wakeUpQueues(i).nonEmpty && wakeup.bits.rfWenCopy.nonEmpty) {
579      wakeup.bits.rfWenCopy.get := wakeUpQueues(i).get.io.deq.bits.rfWenCopy.get
580    }
581    if (wakeUpQueues(i).nonEmpty && wakeup.bits.fpWenCopy.nonEmpty) {
582      wakeup.bits.fpWenCopy.get := wakeUpQueues(i).get.io.deq.bits.fpWenCopy.get
583    }
584    if (wakeUpQueues(i).nonEmpty && wakeup.bits.vecWenCopy.nonEmpty) {
585      wakeup.bits.vecWenCopy.get := wakeUpQueues(i).get.io.deq.bits.vecWenCopy.get
586    }
587    if (wakeUpQueues(i).nonEmpty && wakeup.bits.loadDependencyCopy.nonEmpty) {
588      wakeup.bits.loadDependencyCopy.get := wakeUpQueues(i).get.io.deq.bits.loadDependencyCopy.get
589    }
590  }
591
592  // Todo: better counter implementation
593  private val enqHasValid = validVec.take(params.numEnq).reduce(_ | _)
594  private val enqEntryValidCnt = PopCount(validVec.take(params.numEnq))
595  private val othersValidCnt = PopCount(validVec.drop(params.numEnq))
596  io.status.leftVec(0) := validVec.drop(params.numEnq).reduce(_ & _)
597  for (i <- 0 until params.numEnq) {
598    io.status.leftVec(i + 1) := othersValidCnt === (params.numEntries - params.numEnq - (i + 1)).U
599  }
600  private val othersLeftOneCaseVec = Wire(Vec(params.numEntries - params.numEnq, UInt((params.numEntries - params.numEnq).W)))
601  othersLeftOneCaseVec.zipWithIndex.foreach { case (leftone, i) =>
602    leftone := ~(1.U((params.numEntries - params.numEnq).W) << i)
603  }
604  private val othersLeftOne = othersLeftOneCaseVec.map(_ === VecInit(validVec.drop(params.numEnq)).asUInt).reduce(_ | _)
605  private val othersCanotIn = othersLeftOne || validVec.drop(params.numEnq).reduce(_ & _)
606
607  io.enq.foreach(_.ready := !othersCanotIn || !enqHasValid)
608  io.status.empty := !Cat(validVec).orR
609  io.status.full := othersCanotIn
610  io.status.validCnt := PopCount(validVec)
611
612  protected def getDeqLat(deqPortIdx: Int, fuType: UInt) : UInt = {
613    Mux1H(fuLatencyMaps(deqPortIdx) map { case (k, v) => (fuType(k.id), v.U) })
614  }
615
616  // issue perf counter
617  // enq count
618  XSPerfAccumulate("enq_valid_cnt", PopCount(io.enq.map(_.fire)))
619  XSPerfAccumulate("enq_fire_cnt", PopCount(io.enq.map(_.fire)))
620  // valid count
621  XSPerfHistogram("enq_entry_valid_cnt", enqEntryValidCnt, true.B, 0, params.numEnq + 1)
622  XSPerfHistogram("other_entry_valid_cnt", othersValidCnt, true.B, 0, params.numEntries - params.numEnq + 1)
623  XSPerfHistogram("valid_cnt", PopCount(validVec), true.B, 0, params.numEntries + 1)
624  // only split when more than 1 func type
625  if (params.getFuCfgs.size > 0) {
626    for (t <- FuType.functionNameMap.keys) {
627      val fuName = FuType.functionNameMap(t)
628      if (params.getFuCfgs.map(_.fuType == t).reduce(_ | _)) {
629        XSPerfHistogram(s"valid_cnt_hist_futype_${fuName}", PopCount(validVec.zip(fuTypeVec).map { case (v, fu) => v && fu === t.U }), true.B, 0, params.numEntries, 1)
630      }
631    }
632  }
633  // ready instr count
634  private val readyEntriesCnt = PopCount(validVec.zip(canIssueVec).map(x => x._1 && x._2))
635  XSPerfHistogram("ready_cnt", readyEntriesCnt, true.B, 0, params.numEntries + 1)
636  // only split when more than 1 func type
637  if (params.getFuCfgs.size > 0) {
638    for (t <- FuType.functionNameMap.keys) {
639      val fuName = FuType.functionNameMap(t)
640      if (params.getFuCfgs.map(_.fuType == t).reduce(_ | _)) {
641        XSPerfHistogram(s"ready_cnt_hist_futype_${fuName}", PopCount(validVec.zip(canIssueVec).zip(fuTypeVec).map { case ((v, c), fu) => v && c && fu === t.U }), true.B, 0, params.numEntries, 1)
642      }
643    }
644  }
645
646  // deq instr count
647  XSPerfAccumulate("issue_instr_pre_count", PopCount(deqBeforeDly.map(_.valid)))
648  XSPerfHistogram("issue_instr_pre_count_hist", PopCount(deqBeforeDly.map(_.valid)), true.B, 0, params.numDeq + 1, 1)
649  XSPerfAccumulate("issue_instr_count", PopCount(io.deqDelay.map(_.valid)))
650  XSPerfHistogram("issue_instr_count_hist", PopCount(io.deqDelay.map(_.valid)), true.B, 0, params.numDeq + 1, 1)
651
652  // deq instr data source count
653  XSPerfAccumulate("issue_datasource_reg", deqBeforeDly.map{ deq =>
654    PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.reg && !SrcType.isNotReg(deq.bits.srcType(j)) })
655  }.reduce(_ +& _))
656  XSPerfAccumulate("issue_datasource_bypass", deqBeforeDly.map{ deq =>
657    PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.bypass && !SrcType.isNotReg(deq.bits.srcType(j)) })
658  }.reduce(_ +& _))
659  XSPerfAccumulate("issue_datasource_forward", deqBeforeDly.map{ deq =>
660    PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.forward && !SrcType.isNotReg(deq.bits.srcType(j)) })
661  }.reduce(_ +& _))
662  XSPerfAccumulate("issue_datasource_noreg", deqBeforeDly.map{ deq =>
663    PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && SrcType.isNotReg(deq.bits.srcType(j)) })
664  }.reduce(_ +& _))
665
666  XSPerfHistogram("issue_datasource_reg_hist", deqBeforeDly.map{ deq =>
667    PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.reg && !SrcType.isNotReg(deq.bits.srcType(j)) })
668  }.reduce(_ +& _), true.B, 0, params.numDeq * params.numRegSrc + 1, 1)
669  XSPerfHistogram("issue_datasource_bypass_hist", deqBeforeDly.map{ deq =>
670    PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.bypass && !SrcType.isNotReg(deq.bits.srcType(j)) })
671  }.reduce(_ +& _), true.B, 0, params.numDeq * params.numRegSrc + 1, 1)
672  XSPerfHistogram("issue_datasource_forward_hist", deqBeforeDly.map{ deq =>
673    PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.forward && !SrcType.isNotReg(deq.bits.srcType(j)) })
674  }.reduce(_ +& _), true.B, 0, params.numDeq * params.numRegSrc + 1, 1)
675  XSPerfHistogram("issue_datasource_noreg_hist", deqBeforeDly.map{ deq =>
676    PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && SrcType.isNotReg(deq.bits.srcType(j)) })
677  }.reduce(_ +& _), true.B, 0, params.numDeq * params.numRegSrc + 1, 1)
678
679  // deq instr data source count for each futype
680  for (t <- FuType.functionNameMap.keys) {
681    val fuName = FuType.functionNameMap(t)
682    if (params.getFuCfgs.map(_.fuType == t).reduce(_ | _)) {
683      XSPerfAccumulate(s"issue_datasource_reg_futype_${fuName}", deqBeforeDly.map{ deq =>
684        PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.reg && !SrcType.isNotReg(deq.bits.srcType(j)) && deq.bits.common.fuType === t.U })
685      }.reduce(_ +& _))
686      XSPerfAccumulate(s"issue_datasource_bypass_futype_${fuName}", deqBeforeDly.map{ deq =>
687        PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.bypass && !SrcType.isNotReg(deq.bits.srcType(j)) && deq.bits.common.fuType === t.U })
688      }.reduce(_ +& _))
689      XSPerfAccumulate(s"issue_datasource_forward_futype_${fuName}", deqBeforeDly.map{ deq =>
690        PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.forward && !SrcType.isNotReg(deq.bits.srcType(j)) && deq.bits.common.fuType === t.U })
691      }.reduce(_ +& _))
692      XSPerfAccumulate(s"issue_datasource_noreg_futype_${fuName}", deqBeforeDly.map{ deq =>
693        PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && SrcType.isNotReg(deq.bits.srcType(j)) && deq.bits.common.fuType === t.U })
694      }.reduce(_ +& _))
695
696      XSPerfHistogram(s"issue_datasource_reg_hist_futype_${fuName}", deqBeforeDly.map{ deq =>
697        PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.reg && !SrcType.isNotReg(deq.bits.srcType(j)) && deq.bits.common.fuType === t.U })
698      }.reduce(_ +& _), true.B, 0, params.numDeq * params.numRegSrc + 1, 1)
699      XSPerfHistogram(s"issue_datasource_bypass_hist_futype_${fuName}", deqBeforeDly.map{ deq =>
700        PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.bypass && !SrcType.isNotReg(deq.bits.srcType(j)) && deq.bits.common.fuType === t.U })
701      }.reduce(_ +& _), true.B, 0, params.numDeq * params.numRegSrc + 1, 1)
702      XSPerfHistogram(s"issue_datasource_forward_hist_futype_${fuName}", deqBeforeDly.map{ deq =>
703        PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && ds.value === DataSource.forward && !SrcType.isNotReg(deq.bits.srcType(j)) && deq.bits.common.fuType === t.U })
704      }.reduce(_ +& _), true.B, 0, params.numDeq * params.numRegSrc + 1, 1)
705      XSPerfHistogram(s"issue_datasource_noreg_hist_futype_${fuName}", deqBeforeDly.map{ deq =>
706        PopCount(deq.bits.common.dataSources.zipWithIndex.map{ case (ds, j) => deq.valid && SrcType.isNotReg(deq.bits.srcType(j)) && deq.bits.common.fuType === t.U })
707      }.reduce(_ +& _), true.B, 0, params.numDeq * params.numRegSrc + 1, 1)
708    }
709  }
710
711  // cancel instr count
712  if (params.hasIQWakeUp) {
713    val cancelVec: Vec[Bool] = entries.io.cancel.get
714    XSPerfAccumulate("cancel_instr_count", PopCount(validVec.zip(cancelVec).map(x => x._1 & x._2)))
715    XSPerfHistogram("cancel_instr_hist", PopCount(validVec.zip(cancelVec).map(x => x._1 & x._2)), true.B, 0, params.numEntries, 1)
716    for (t <- FuType.functionNameMap.keys) {
717      val fuName = FuType.functionNameMap(t)
718      if (params.getFuCfgs.map(_.fuType == t).reduce(_ | _)) {
719        XSPerfAccumulate(s"cancel_instr_count_futype_${fuName}", PopCount(validVec.zip(cancelVec).zip(fuTypeVec).map{ case ((x, y), fu) => x & y & fu === t.U }))
720        XSPerfHistogram(s"cancel_instr_hist_futype_${fuName}", PopCount(validVec.zip(cancelVec).zip(fuTypeVec).map{ case ((x, y), fu) => x & y & fu === t.U }), true.B, 0, params.numEntries, 1)
721      }
722    }
723  }
724}
725
726class IssueQueueJumpBundle extends Bundle {
727  val pc = UInt(VAddrData().dataWidth.W)
728}
729
730class IssueQueueLoadBundle(implicit p: Parameters) extends XSBundle {
731  val fastMatch = UInt(backendParams.LduCnt.W)
732  val fastImm = UInt(12.W)
733}
734
735class IssueQueueIntIO()(implicit p: Parameters, params: IssueBlockParams) extends IssueQueueIO
736
737class IssueQueueIntImp(override val wrapper: IssueQueue)(implicit p: Parameters, iqParams: IssueBlockParams)
738  extends IssueQueueImp(wrapper)
739{
740  io.suggestName("none")
741  override lazy val io = IO(new IssueQueueIntIO).suggestName("io")
742
743  deqBeforeDly.zipWithIndex.foreach{ case (deq, i) => {
744    deq.bits.common.pc.foreach(_ := deqEntryVec(i).bits.payload.pc)
745    deq.bits.common.preDecode.foreach(_ := deqEntryVec(i).bits.payload.preDecodeInfo)
746    deq.bits.common.ftqIdx.foreach(_ := deqEntryVec(i).bits.payload.ftqPtr)
747    deq.bits.common.ftqOffset.foreach(_ := deqEntryVec(i).bits.payload.ftqOffset)
748    deq.bits.common.predictInfo.foreach(x => {
749      x.target := DontCare
750      x.taken := deqEntryVec(i).bits.payload.pred_taken
751    })
752    // for std
753    deq.bits.common.sqIdx.foreach(_ := deqEntryVec(i).bits.payload.sqIdx)
754    // for i2f
755    deq.bits.common.fpu.foreach(_ := deqEntryVec(i).bits.payload.fpu)
756  }}
757}
758
759class IssueQueueVfImp(override val wrapper: IssueQueue)(implicit p: Parameters, iqParams: IssueBlockParams)
760  extends IssueQueueImp(wrapper)
761{
762  s0_enqBits.foreach{ x =>
763    x.srcType(3) := SrcType.vp // v0: mask src
764    x.srcType(4) := SrcType.vp // vl&vtype
765  }
766  deqBeforeDly.zipWithIndex.foreach{ case (deq, i) => {
767    deq.bits.common.fpu.foreach(_ := deqEntryVec(i).bits.payload.fpu)
768    deq.bits.common.vpu.foreach(_ := deqEntryVec(i).bits.payload.vpu)
769    deq.bits.common.vpu.foreach(_.vuopIdx := deqEntryVec(i).bits.payload.uopIdx)
770    deq.bits.common.vpu.foreach(_.lastUop := deqEntryVec(i).bits.payload.lastUop)
771  }}
772}
773
774class IssueQueueMemBundle(implicit p: Parameters, params: IssueBlockParams) extends Bundle {
775  val feedbackIO = Flipped(Vec(params.numDeq, new MemRSFeedbackIO))
776  val checkWait = new Bundle {
777    val stIssuePtr = Input(new SqPtr)
778    val memWaitUpdateReq = Flipped(new MemWaitUpdateReq)
779  }
780  val loadFastMatch = Output(Vec(params.LdExuCnt, new IssueQueueLoadBundle))
781  val loadWakeUp = Input(Vec(params.LdExuCnt, ValidIO(new DynInst())))
782
783  // vector
784  val sqDeqPtr = OptionWrapper(params.isVecMemIQ, Input(new SqPtr))
785  val lqDeqPtr = OptionWrapper(params.isVecMemIQ, Input(new LqPtr))
786}
787
788class IssueQueueMemIO(implicit p: Parameters, params: IssueBlockParams) extends IssueQueueIO {
789  val memIO = Some(new IssueQueueMemBundle)
790}
791
792class IssueQueueMemAddrImp(override val wrapper: IssueQueue)(implicit p: Parameters, params: IssueBlockParams)
793  extends IssueQueueImp(wrapper) with HasCircularQueuePtrHelper {
794
795  require(params.StdCnt == 0 && (params.LduCnt + params.StaCnt + params.HyuCnt + params.VlduCnt) > 0, "IssueQueueMemAddrImp can only be instance of MemAddr IQ, " +
796    s"StdCnt: ${params.StdCnt}, LduCnt: ${params.LduCnt}, StaCnt: ${params.StaCnt}, HyuCnt: ${params.HyuCnt}")
797  println(s"[IssueQueueMemAddrImp] StdCnt: ${params.StdCnt}, LduCnt: ${params.LduCnt}, StaCnt: ${params.StaCnt}, HyuCnt: ${params.HyuCnt}")
798
799  io.suggestName("none")
800  override lazy val io = IO(new IssueQueueMemIO).suggestName("io")
801  private val memIO = io.memIO.get
802
803  memIO.loadFastMatch := 0.U.asTypeOf(memIO.loadFastMatch) // TODO: is still needed?
804
805  for (i <- io.enq.indices) {
806    val blockNotReleased = isAfter(io.enq(i).bits.sqIdx, memIO.checkWait.stIssuePtr)
807    val storeAddrWaitForIsIssuing = VecInit((0 until StorePipelineWidth).map(i => {
808      memIO.checkWait.memWaitUpdateReq.robIdx(i).valid &&
809        memIO.checkWait.memWaitUpdateReq.robIdx(i).bits.value === io.enq(i).bits.waitForRobIdx.value
810    })).asUInt.orR && !io.enq(i).bits.loadWaitStrict // is waiting for store addr ready
811    s0_enqBits(i).loadWaitBit := io.enq(i).bits.loadWaitBit && !storeAddrWaitForIsIssuing && blockNotReleased
812    // when have vpu
813    if (params.VlduCnt > 0 || params.VstuCnt > 0) {
814      s0_enqBits(i).srcType(3) := SrcType.vp // v0: mask src
815      s0_enqBits(i).srcType(4) := SrcType.vp // vl&vtype
816    }
817  }
818
819  for (i <- entries.io.enq.indices) {
820    entries.io.enq(i).bits.status match { case enqData =>
821      enqData.blocked := false.B // s0_enqBits(i).loadWaitBit
822      enqData.mem.get.strictWait := s0_enqBits(i).loadWaitStrict
823      enqData.mem.get.waitForStd := false.B
824      enqData.mem.get.waitForRobIdx := s0_enqBits(i).waitForRobIdx
825      enqData.mem.get.waitForSqIdx := 0.U.asTypeOf(enqData.mem.get.waitForSqIdx) // generated by sq, will be updated later
826      enqData.mem.get.sqIdx := s0_enqBits(i).sqIdx
827    }
828  }
829  entries.io.fromMem.get.slowResp.zipWithIndex.foreach { case (slowResp, i) =>
830    slowResp.valid := memIO.feedbackIO(i).feedbackSlow.valid
831    slowResp.bits.robIdx := memIO.feedbackIO(i).feedbackSlow.bits.robIdx
832    slowResp.bits.respType := Mux(memIO.feedbackIO(i).feedbackSlow.bits.hit, RSFeedbackType.fuIdle, RSFeedbackType.feedbackInvalid)
833    slowResp.bits.dataInvalidSqIdx := memIO.feedbackIO(i).feedbackSlow.bits.dataInvalidSqIdx
834    slowResp.bits.rfWen := DontCare
835    slowResp.bits.fuType := DontCare
836  }
837
838  entries.io.fromMem.get.fastResp.zipWithIndex.foreach { case (fastResp, i) =>
839    fastResp.valid := memIO.feedbackIO(i).feedbackFast.valid
840    fastResp.bits.robIdx := memIO.feedbackIO(i).feedbackFast.bits.robIdx
841    fastResp.bits.respType := Mux(memIO.feedbackIO(i).feedbackFast.bits.hit, RSFeedbackType.fuIdle, memIO.feedbackIO(i).feedbackFast.bits.sourceType)
842    fastResp.bits.dataInvalidSqIdx := 0.U.asTypeOf(fastResp.bits.dataInvalidSqIdx)
843    fastResp.bits.rfWen := DontCare
844    fastResp.bits.fuType := DontCare
845  }
846
847  entries.io.fromMem.get.memWaitUpdateReq := memIO.checkWait.memWaitUpdateReq
848  entries.io.fromMem.get.stIssuePtr := memIO.checkWait.stIssuePtr
849
850  // load wakeup
851  val loadWakeUpIter = memIO.loadWakeUp.iterator
852  io.wakeupToIQ.zip(params.exuBlockParams).zipWithIndex.foreach { case ((wakeup, param), i) =>
853    if (param.hasLoadExu) {
854      require(wakeUpQueues(i).isEmpty)
855      val uop = loadWakeUpIter.next()
856
857      wakeup.valid := RegNext(uop.fire)
858      wakeup.bits.rfWen  := RegNext(uop.bits.rfWen  && uop.fire)
859      wakeup.bits.fpWen  := RegNext(uop.bits.fpWen  && uop.fire)
860      wakeup.bits.vecWen := RegNext(uop.bits.vecWen && uop.fire)
861      wakeup.bits.pdest  := RegNext(uop.bits.pdest)
862      wakeup.bits.loadDependency.foreach(_ := 0.U) // this is correct for load only
863
864      wakeup.bits.rfWenCopy .foreach(_.foreach(_ := RegNext(uop.bits.rfWen  && uop.fire)))
865      wakeup.bits.fpWenCopy .foreach(_.foreach(_ := RegNext(uop.bits.fpWen  && uop.fire)))
866      wakeup.bits.vecWenCopy.foreach(_.foreach(_ := RegNext(uop.bits.vecWen && uop.fire)))
867      wakeup.bits.pdestCopy .foreach(_.foreach(_ := RegNext(uop.bits.pdest)))
868      wakeup.bits.loadDependencyCopy.foreach(x => x := 0.U.asTypeOf(x)) // this is correct for load only
869
870      wakeup.bits.is0Lat := 0.U
871    }
872  }
873  require(!loadWakeUpIter.hasNext)
874
875  deqBeforeDly.zipWithIndex.foreach { case (deq, i) =>
876    deq.bits.common.loadWaitBit.foreach(_ := deqEntryVec(i).bits.payload.loadWaitBit)
877    deq.bits.common.waitForRobIdx.foreach(_ := deqEntryVec(i).bits.payload.waitForRobIdx)
878    deq.bits.common.storeSetHit.foreach(_ := deqEntryVec(i).bits.payload.storeSetHit)
879    deq.bits.common.loadWaitStrict.foreach(_ := deqEntryVec(i).bits.payload.loadWaitStrict)
880    deq.bits.common.ssid.foreach(_ := deqEntryVec(i).bits.payload.ssid)
881    deq.bits.common.sqIdx.get := deqEntryVec(i).bits.payload.sqIdx
882    deq.bits.common.lqIdx.get := deqEntryVec(i).bits.payload.lqIdx
883    deq.bits.common.ftqIdx.foreach(_ := deqEntryVec(i).bits.payload.ftqPtr)
884    deq.bits.common.ftqOffset.foreach(_ := deqEntryVec(i).bits.payload.ftqOffset)
885    // when have vpu
886    if (params.VlduCnt > 0 || params.VstuCnt > 0) {
887      deq.bits.common.vpu.foreach(_ := deqEntryVec(i).bits.payload.vpu)
888      deq.bits.common.vpu.foreach(_.vuopIdx := deqEntryVec(i).bits.payload.uopIdx)
889    }
890  }
891}
892
893class IssueQueueVecMemImp(override val wrapper: IssueQueue)(implicit p: Parameters, params: IssueBlockParams)
894  extends IssueQueueImp(wrapper) with HasCircularQueuePtrHelper {
895
896  require((params.VstdCnt + params.VlduCnt + params.VstaCnt) > 0, "IssueQueueVecMemImp can only be instance of VecMem IQ")
897
898  io.suggestName("none")
899  override lazy val io = IO(new IssueQueueMemIO).suggestName("io")
900  private val memIO = io.memIO.get
901
902  def selectOldUop(robIdx: Seq[RobPtr], uopIdx: Seq[UInt], valid: Seq[Bool]): Vec[Bool] = {
903    val compareVec = (0 until robIdx.length).map(i => (0 until i).map(j => isAfter(robIdx(j), robIdx(i)) || (robIdx(j).value === robIdx(i).value && uopIdx(i) < uopIdx(j))))
904    val resultOnehot = VecInit((0 until robIdx.length).map(i => Cat((0 until robIdx.length).map(j =>
905      (if (j < i) !valid(j) || compareVec(i)(j)
906      else if (j == i) valid(i)
907      else !valid(j) || !compareVec(j)(i))
908    )).andR))
909    resultOnehot
910  }
911
912  val robIdxVec = entries.io.robIdx.get
913  val uopIdxVec = entries.io.uopIdx.get
914  val allEntryOldestOH = selectOldUop(robIdxVec, uopIdxVec, validVec)
915
916  finalDeqSelValidVec.head := (allEntryOldestOH.asUInt & canIssueVec.asUInt).orR
917  finalDeqSelOHVec.head := allEntryOldestOH.asUInt & canIssueVec.asUInt
918
919  if (params.isVecMemAddrIQ) {
920    s0_enqBits.foreach{ x =>
921      x.srcType(3) := SrcType.vp // v0: mask src
922      x.srcType(4) := SrcType.vp // vl&vtype
923    }
924
925    for (i <- io.enq.indices) {
926      s0_enqBits(i).loadWaitBit := false.B
927    }
928
929    for (i <- entries.io.enq.indices) {
930      entries.io.enq(i).bits.status match { case enqData =>
931        enqData.blocked := false.B // s0_enqBits(i).loadWaitBit
932        enqData.mem.get.strictWait := s0_enqBits(i).loadWaitStrict
933        enqData.mem.get.waitForStd := false.B
934        enqData.mem.get.waitForRobIdx := s0_enqBits(i).waitForRobIdx
935        enqData.mem.get.waitForSqIdx := 0.U.asTypeOf(enqData.mem.get.waitForSqIdx) // generated by sq, will be updated later
936        enqData.mem.get.sqIdx := s0_enqBits(i).sqIdx
937      }
938
939      entries.io.fromMem.get.slowResp.zipWithIndex.foreach { case (slowResp, i) =>
940        slowResp.valid                 := memIO.feedbackIO(i).feedbackSlow.valid
941        slowResp.bits.robIdx           := memIO.feedbackIO(i).feedbackSlow.bits.robIdx
942        slowResp.bits.respType         := Mux(memIO.feedbackIO(i).feedbackSlow.bits.hit, RSFeedbackType.fuIdle, RSFeedbackType.feedbackInvalid)
943        slowResp.bits.dataInvalidSqIdx := memIO.feedbackIO(i).feedbackSlow.bits.dataInvalidSqIdx
944        slowResp.bits.rfWen := DontCare
945        slowResp.bits.fuType := DontCare
946      }
947
948      entries.io.fromMem.get.fastResp.zipWithIndex.foreach { case (fastResp, i) =>
949        fastResp.valid                 := memIO.feedbackIO(i).feedbackFast.valid
950        fastResp.bits.robIdx           := memIO.feedbackIO(i).feedbackFast.bits.robIdx
951        fastResp.bits.respType         := memIO.feedbackIO(i).feedbackFast.bits.sourceType
952        fastResp.bits.dataInvalidSqIdx := 0.U.asTypeOf(fastResp.bits.dataInvalidSqIdx)
953        fastResp.bits.rfWen := DontCare
954        fastResp.bits.fuType := DontCare
955      }
956
957      entries.io.fromMem.get.memWaitUpdateReq := memIO.checkWait.memWaitUpdateReq
958      entries.io.fromMem.get.stIssuePtr := memIO.checkWait.stIssuePtr
959    }
960  }
961
962  for (i <- entries.io.enq.indices) {
963    entries.io.enq(i).bits.status.vecMem.get match {
964      case enqData =>
965        enqData.sqIdx := s0_enqBits(i).sqIdx
966        enqData.lqIdx := s0_enqBits(i).lqIdx
967        enqData.uopIdx := s0_enqBits(i).uopIdx
968    }
969  }
970
971  entries.io.vecMemIn.get.sqDeqPtr := memIO.sqDeqPtr.get
972  entries.io.vecMemIn.get.lqDeqPtr := memIO.lqDeqPtr.get
973
974  entries.io.fromMem.get.fastResp.zipWithIndex.foreach { case (resp, i) =>
975    resp.bits.uopIdx.get := 0.U // Todo
976  }
977
978  entries.io.fromMem.get.slowResp.zipWithIndex.foreach { case (resp, i) =>
979    resp.bits.uopIdx.get := 0.U // Todo
980  }
981
982  deqBeforeDly.zipWithIndex.foreach { case (deq, i) =>
983    deq.bits.common.sqIdx.foreach(_ := deqEntryVec(i).bits.payload.sqIdx)
984    deq.bits.common.lqIdx.foreach(_ := deqEntryVec(i).bits.payload.lqIdx)
985    if (params.isVecLdAddrIQ) {
986      deq.bits.common.ftqIdx.get := deqEntryVec(i).bits.payload.ftqPtr
987      deq.bits.common.ftqOffset.get := deqEntryVec(i).bits.payload.ftqOffset
988    }
989    deq.bits.common.fpu.foreach(_ := deqEntryVec(i).bits.payload.fpu)
990    deq.bits.common.vpu.foreach(_ := deqEntryVec(i).bits.payload.vpu)
991    deq.bits.common.vpu.foreach(_.vuopIdx := deqEntryVec(i).bits.payload.uopIdx)
992    deq.bits.common.vpu.foreach(_.lastUop := deqEntryVec(i).bits.payload.lastUop)
993  }
994}
995