xref: /XiangShan/src/main/scala/xiangshan/mem/MemCommon.scala (revision ad415ae048fdc4bc9928dd381489cc92da3ca4f9) 
1/***************************************************************************************
2* Copyright (c) 2024 Beijing Institute of Open Source Chip (BOSC)
3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences
4* Copyright (c) 2020-2021 Peng Cheng Laboratory
5*
6* XiangShan is licensed under Mulan PSL v2.
7* You can use this software according to the terms and conditions of the Mulan PSL v2.
8* You may obtain a copy of Mulan PSL v2 at:
9*          http://license.coscl.org.cn/MulanPSL2
10*
11* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14*
15* See the Mulan PSL v2 for more details.
16***************************************************************************************/
17
18package xiangshan.mem
19
20
21import org.chipsalliance.cde.config.Parameters
22import chisel3._
23import chisel3.util._
24import utility._
25import utils._
26import xiangshan._
27import xiangshan.backend.Bundles.{DynInst, MemExuInput}
28import xiangshan.backend.rob.RobPtr
29import xiangshan.cache._
30import xiangshan.backend.fu.FenceToSbuffer
31import xiangshan.cache.wpu.ReplayCarry
32import xiangshan.mem.prefetch.PrefetchReqBundle
33import math._
34
35object genWmask {
36  def apply(addr: UInt, sizeEncode: UInt): UInt = {
37    (LookupTree(sizeEncode, List(
38      "b00".U -> 0x1.U, //0001 << addr(2:0)
39      "b01".U -> 0x3.U, //0011
40      "b10".U -> 0xf.U, //1111
41      "b11".U -> 0xff.U //11111111
42    )) << addr(2, 0)).asUInt
43  }
44}
45
46object genVWmask {
47  def apply(addr: UInt, sizeEncode: UInt): UInt = {
48    (LookupTree(sizeEncode, List(
49      "b00".U -> 0x1.U, //0001 << addr(2:0)
50      "b01".U -> 0x3.U, //0011
51      "b10".U -> 0xf.U, //1111
52      "b11".U -> 0xff.U //11111111
53    )) << addr(3, 0)).asUInt
54  }
55}
56
57object genWdata {
58  def apply(data: UInt, sizeEncode: UInt): UInt = {
59    LookupTree(sizeEncode, List(
60      "b00".U -> Fill(16, data(7, 0)),
61      "b01".U -> Fill(8, data(15, 0)),
62      "b10".U -> Fill(4, data(31, 0)),
63      "b11".U -> Fill(2, data(63,0))
64    ))
65  }
66}
67
68object shiftDataToLow {
69  def apply(addr: UInt,data : UInt): UInt = {
70    Mux(addr(3), (data >> 64).asUInt,data)
71  }
72}
73object shiftMaskToLow {
74  def apply(addr: UInt,mask: UInt): UInt = {
75    Mux(addr(3),(mask >> 8).asUInt,mask)
76  }
77}
78
79class LsPipelineBundle(implicit p: Parameters) extends XSBundle
80  with HasDCacheParameters
81  with HasVLSUParameters {
82  val uop = new DynInst
83  val vaddr = UInt(VAddrBits.W)
84  val fullva = UInt(XLEN.W)
85  val paddr = UInt(PAddrBits.W)
86  val gpaddr = UInt(XLEN.W)
87  val isForVSnonLeafPTE = Bool()
88  // val func = UInt(6.W)
89  val mask = UInt((VLEN/8).W)
90  val data = UInt((VLEN+1).W)
91  val wlineflag = Bool() // store write the whole cache line
92
93  val miss = Bool()
94  val tlbMiss = Bool()
95  val ptwBack = Bool()
96  val af = Bool()
97  val mmio = Bool()
98  val atomic = Bool()
99
100  val forwardMask = Vec(VLEN/8, Bool())
101  val forwardData = Vec(VLEN/8, UInt(8.W))
102
103  // prefetch
104  val isPrefetch = Bool()
105  val isHWPrefetch = Bool()
106  def isSWPrefetch = isPrefetch && !isHWPrefetch
107
108  // misalignBuffer
109  val isFrmMisAlignBuf = Bool()
110
111  // vector
112  val isvec = Bool()
113  val isLastElem = Bool()
114  val is128bit = Bool()
115  val uop_unit_stride_fof = Bool()
116  val usSecondInv = Bool()
117  val elemIdx = UInt(elemIdxBits.W)
118  val alignedType = UInt(alignTypeBits.W)
119  val mbIndex = UInt(max(vlmBindexBits, vsmBindexBits).W)
120  // val rob_idx_valid = Vec(2,Bool())
121  // val inner_idx = Vec(2,UInt(3.W))
122  // val rob_idx = Vec(2,new RobPtr)
123  val reg_offset = UInt(vOffsetBits.W)
124  val elemIdxInsideVd = UInt(elemIdxBits.W)
125  // val offset = Vec(2,UInt(4.W))
126  val vecActive = Bool() // 1: vector active element or scala mem operation, 0: vector not active element
127  val is_first_ele = Bool()
128  // val flowPtr = new VlflowPtr() // VLFlowQueue ptr
129  // val sflowPtr = new VsFlowPtr() // VSFlowQueue ptr
130
131  // For debug usage
132  val isFirstIssue = Bool()
133  val hasROBEntry = Bool()
134
135  // For load replay
136  val isLoadReplay = Bool()
137  val isFastPath = Bool()
138  val isFastReplay = Bool()
139  val replayCarry = new ReplayCarry(nWays)
140
141  // For dcache miss load
142  val mshrid = UInt(log2Up(cfg.nMissEntries).W)
143  val handledByMSHR = Bool()
144  val replacementUpdated = Bool()
145  val missDbUpdated = Bool()
146
147  val forward_tlDchannel = Bool()
148  val dcacheRequireReplay = Bool()
149  val delayedLoadError = Bool()
150  val lateKill = Bool()
151  val feedbacked = Bool()
152  val ldCancel = ValidUndirectioned(UInt(log2Ceil(LoadPipelineWidth).W))
153  // loadQueueReplay index.
154  val schedIndex = UInt(log2Up(LoadQueueReplaySize).W)
155  // hardware prefetch and fast replay no need to query tlb
156  val tlbNoQuery = Bool()
157}
158
159class LdPrefetchTrainBundle(implicit p: Parameters) extends LsPipelineBundle {
160  val meta_prefetch = UInt(L1PfSourceBits.W)
161  val meta_access = Bool()
162
163  def fromLsPipelineBundle(input: LsPipelineBundle, latch: Boolean = false, enable: Bool = true.B) = {
164    if (latch) vaddr := RegEnable(input.vaddr, enable) else vaddr := input.vaddr
165    if (latch) fullva := RegEnable(input.fullva, enable) else fullva := input.fullva
166    if (latch) paddr := RegEnable(input.paddr, enable) else paddr := input.paddr
167    if (latch) gpaddr := RegEnable(input.gpaddr, enable) else gpaddr := input.gpaddr
168    if (latch) isForVSnonLeafPTE := RegEnable(input.isForVSnonLeafPTE, enable) else isForVSnonLeafPTE := input.isForVSnonLeafPTE
169    if (latch) mask := RegEnable(input.mask, enable) else mask := input.mask
170    if (latch) data := RegEnable(input.data, enable) else data := input.data
171    if (latch) uop := RegEnable(input.uop, enable) else uop := input.uop
172    if (latch) wlineflag := RegEnable(input.wlineflag, enable) else wlineflag := input.wlineflag
173    if (latch) miss := RegEnable(input.miss, enable) else miss := input.miss
174    if (latch) tlbMiss := RegEnable(input.tlbMiss, enable) else tlbMiss := input.tlbMiss
175    if (latch) ptwBack := RegEnable(input.ptwBack, enable) else ptwBack := input.ptwBack
176    if (latch) af := RegEnable(input.af, enable) else af := input.af
177    if (latch) mmio := RegEnable(input.mmio, enable) else mmio := input.mmio
178    if (latch) forwardMask := RegEnable(input.forwardMask, enable) else forwardMask := input.forwardMask
179    if (latch) forwardData := RegEnable(input.forwardData, enable) else forwardData := input.forwardData
180    if (latch) isPrefetch := RegEnable(input.isPrefetch, enable) else isPrefetch := input.isPrefetch
181    if (latch) isHWPrefetch := RegEnable(input.isHWPrefetch, enable) else isHWPrefetch := input.isHWPrefetch
182    if (latch) isFrmMisAlignBuf := RegEnable(input.isFrmMisAlignBuf, enable) else isFrmMisAlignBuf := input.isFrmMisAlignBuf
183    if (latch) isFirstIssue := RegEnable(input.isFirstIssue, enable) else isFirstIssue := input.isFirstIssue
184    if (latch) hasROBEntry := RegEnable(input.hasROBEntry, enable) else hasROBEntry := input.hasROBEntry
185    if (latch) dcacheRequireReplay := RegEnable(input.dcacheRequireReplay, enable) else dcacheRequireReplay := input.dcacheRequireReplay
186    if (latch) schedIndex := RegEnable(input.schedIndex, enable) else schedIndex := input.schedIndex
187    if (latch) tlbNoQuery := RegEnable(input.tlbNoQuery, enable) else tlbNoQuery := input.tlbNoQuery
188    if (latch) isvec               := RegEnable(input.isvec, enable)               else isvec               := input.isvec
189    if (latch) isLastElem          := RegEnable(input.isLastElem, enable)          else isLastElem          := input.isLastElem
190    if (latch) is128bit            := RegEnable(input.is128bit, enable)            else is128bit            := input.is128bit
191    if (latch) vecActive           := RegEnable(input.vecActive, enable)           else vecActive           := input.vecActive
192    if (latch) is_first_ele        := RegEnable(input.is_first_ele, enable)        else is_first_ele        := input.is_first_ele
193    if (latch) uop_unit_stride_fof := RegEnable(input.uop_unit_stride_fof, enable) else uop_unit_stride_fof := input.uop_unit_stride_fof
194    if (latch) usSecondInv         := RegEnable(input.usSecondInv, enable)         else usSecondInv         := input.usSecondInv
195    if (latch) reg_offset          := RegEnable(input.reg_offset, enable)          else reg_offset          := input.reg_offset
196    if (latch) elemIdx             := RegEnable(input.elemIdx, enable)             else elemIdx             := input.elemIdx
197    if (latch) alignedType         := RegEnable(input.alignedType, enable)         else alignedType         := input.alignedType
198    if (latch) mbIndex             := RegEnable(input.mbIndex, enable)             else mbIndex             := input.mbIndex
199    if (latch) elemIdxInsideVd     := RegEnable(input.elemIdxInsideVd, enable)     else elemIdxInsideVd     := input.elemIdxInsideVd
200    // if (latch) flowPtr             := RegEnable(input.flowPtr, enable)             else flowPtr             := input.flowPtr
201    // if (latch) sflowPtr            := RegEnable(input.sflowPtr, enable)            else sflowPtr            := input.sflowPtr
202
203    meta_prefetch := DontCare
204    meta_access := DontCare
205    forward_tlDchannel := DontCare
206    mshrid := DontCare
207    replayCarry := DontCare
208    atomic := DontCare
209    isLoadReplay := DontCare
210    isFastPath := DontCare
211    isFastReplay := DontCare
212    handledByMSHR := DontCare
213    replacementUpdated := DontCare
214    missDbUpdated := DontCare
215    delayedLoadError := DontCare
216    lateKill := DontCare
217    feedbacked := DontCare
218    ldCancel := DontCare
219  }
220
221  def asPrefetchReqBundle(): PrefetchReqBundle = {
222    val res = Wire(new PrefetchReqBundle)
223    res.vaddr       := this.vaddr
224    res.paddr       := this.paddr
225    res.pc          := this.uop.pc
226    res.miss        := this.miss
227    res.pfHitStream := isFromStream(this.meta_prefetch)
228
229    res
230  }
231}
232
233class StPrefetchTrainBundle(implicit p: Parameters) extends LdPrefetchTrainBundle {}
234
235class LqWriteBundle(implicit p: Parameters) extends LsPipelineBundle {
236  // load inst replay informations
237  val rep_info = new LoadToLsqReplayIO
238  // queue entry data, except flag bits, will be updated if writeQueue is true,
239  // valid bit in LqWriteBundle will be ignored
240  val data_wen_dup = Vec(6, Bool()) // dirty reg dup
241
242
243  def fromLsPipelineBundle(input: LsPipelineBundle, latch: Boolean = false, enable: Bool = true.B) = {
244    if(latch) vaddr := RegEnable(input.vaddr, enable) else vaddr := input.vaddr
245    if(latch) fullva := RegEnable(input.fullva, enable) else fullva := input.fullva
246    if(latch) paddr := RegEnable(input.paddr, enable) else paddr := input.paddr
247    if(latch) gpaddr := RegEnable(input.gpaddr, enable) else gpaddr := input.gpaddr
248    if(latch) isForVSnonLeafPTE := RegEnable(input.isForVSnonLeafPTE, enable) else isForVSnonLeafPTE := input.isForVSnonLeafPTE
249    if(latch) mask := RegEnable(input.mask, enable) else mask := input.mask
250    if(latch) data := RegEnable(input.data, enable) else data := input.data
251    if(latch) uop := RegEnable(input.uop, enable) else uop := input.uop
252    if(latch) wlineflag := RegEnable(input.wlineflag, enable) else wlineflag := input.wlineflag
253    if(latch) miss := RegEnable(input.miss, enable) else miss := input.miss
254    if(latch) tlbMiss := RegEnable(input.tlbMiss, enable) else tlbMiss := input.tlbMiss
255    if(latch) ptwBack := RegEnable(input.ptwBack, enable) else ptwBack := input.ptwBack
256    if(latch) mmio := RegEnable(input.mmio, enable) else mmio := input.mmio
257    if(latch) atomic := RegEnable(input.atomic, enable) else atomic := input.atomic
258    if(latch) forwardMask := RegEnable(input.forwardMask, enable) else forwardMask := input.forwardMask
259    if(latch) forwardData := RegEnable(input.forwardData, enable) else forwardData := input.forwardData
260    if(latch) isPrefetch := RegEnable(input.isPrefetch, enable) else isPrefetch := input.isPrefetch
261    if(latch) isHWPrefetch := RegEnable(input.isHWPrefetch, enable) else isHWPrefetch := input.isHWPrefetch
262    if(latch) isFrmMisAlignBuf := RegEnable(input.isFrmMisAlignBuf, enable) else isFrmMisAlignBuf := input.isFrmMisAlignBuf
263    if(latch) isFirstIssue := RegEnable(input.isFirstIssue, enable) else isFirstIssue := input.isFirstIssue
264    if(latch) hasROBEntry := RegEnable(input.hasROBEntry, enable) else hasROBEntry := input.hasROBEntry
265    if(latch) isLoadReplay := RegEnable(input.isLoadReplay, enable) else isLoadReplay := input.isLoadReplay
266    if(latch) isFastPath := RegEnable(input.isFastPath, enable) else isFastPath := input.isFastPath
267    if(latch) isFastReplay := RegEnable(input.isFastReplay, enable) else isFastReplay := input.isFastReplay
268    if(latch) mshrid := RegEnable(input.mshrid, enable) else mshrid := input.mshrid
269    if(latch) forward_tlDchannel := RegEnable(input.forward_tlDchannel, enable) else forward_tlDchannel := input.forward_tlDchannel
270    if(latch) replayCarry := RegEnable(input.replayCarry, enable) else replayCarry := input.replayCarry
271    if(latch) dcacheRequireReplay := RegEnable(input.dcacheRequireReplay, enable) else dcacheRequireReplay := input.dcacheRequireReplay
272    if(latch) schedIndex := RegEnable(input.schedIndex, enable) else schedIndex := input.schedIndex
273    if(latch) handledByMSHR := RegEnable(input.handledByMSHR, enable) else handledByMSHR := input.handledByMSHR
274    if(latch) replacementUpdated := RegEnable(input.replacementUpdated, enable) else replacementUpdated := input.replacementUpdated
275    if(latch) missDbUpdated := RegEnable(input.missDbUpdated, enable) else missDbUpdated := input.missDbUpdated
276    if(latch) delayedLoadError := RegEnable(input.delayedLoadError, enable) else delayedLoadError := input.delayedLoadError
277    if(latch) lateKill := RegEnable(input.lateKill, enable) else lateKill := input.lateKill
278    if(latch) feedbacked := RegEnable(input.feedbacked, enable) else feedbacked := input.feedbacked
279    if(latch) isvec               := RegEnable(input.isvec, enable)               else isvec               := input.isvec
280    if(latch) is128bit            := RegEnable(input.is128bit, enable)            else is128bit            := input.is128bit
281    if(latch) vecActive           := RegEnable(input.vecActive, enable)           else vecActive           := input.vecActive
282    if(latch) uop_unit_stride_fof := RegEnable(input.uop_unit_stride_fof, enable) else uop_unit_stride_fof := input.uop_unit_stride_fof
283    if(latch) reg_offset          := RegEnable(input.reg_offset, enable)          else reg_offset          := input.reg_offset
284    if(latch) mbIndex             := RegEnable(input.mbIndex, enable)             else mbIndex             := input.mbIndex
285    if(latch) elemIdxInsideVd     := RegEnable(input.elemIdxInsideVd, enable)     else elemIdxInsideVd     := input.elemIdxInsideVd
286
287    rep_info := DontCare
288    data_wen_dup := DontCare
289  }
290}
291
292class SqWriteBundle(implicit p: Parameters) extends LsPipelineBundle {
293  val need_rep = Bool()
294}
295
296class LoadForwardQueryIO(implicit p: Parameters) extends XSBundle {
297  val vaddr = Output(UInt(VAddrBits.W))
298  val paddr = Output(UInt(PAddrBits.W))
299  val mask = Output(UInt((VLEN/8).W))
300  val uop = Output(new DynInst) // for replay
301  val pc = Output(UInt(VAddrBits.W)) //for debug
302  val valid = Output(Bool())
303
304  val forwardMaskFast = Input(Vec((VLEN/8), Bool())) // resp to load_s1
305  val forwardMask = Input(Vec((VLEN/8), Bool())) // resp to load_s2
306  val forwardData = Input(Vec((VLEN/8), UInt(8.W))) // resp to load_s2
307
308  // val lqIdx = Output(UInt(LoadQueueIdxWidth.W))
309  val sqIdx = Output(new SqPtr)
310
311  // dataInvalid suggests store to load forward found forward should happen,
312  // but data is not available for now. If dataInvalid, load inst should
313  // be replayed from RS. Feedback type should be RSFeedbackType.dataInvalid
314  val dataInvalid = Input(Bool()) // Addr match, but data is not valid for now
315
316  // matchInvalid suggests in store to load forward logic, paddr cam result does
317  // to equal to vaddr cam result. If matchInvalid, a microarchitectural exception
318  // should be raised to flush SQ and committed sbuffer.
319  val matchInvalid = Input(Bool()) // resp to load_s2
320
321  // addrInvalid suggests store to load forward found forward should happen,
322  // but address (SSID) is not available for now. If addrInvalid, load inst should
323  // be replayed from RS. Feedback type should be RSFeedbackType.addrInvalid
324  val addrInvalid = Input(Bool())
325}
326
327// LoadForwardQueryIO used in load pipeline
328//
329// Difference between PipeLoadForwardQueryIO and LoadForwardQueryIO:
330// PipeIO use predecoded sqIdxMask for better forward timing
331class PipeLoadForwardQueryIO(implicit p: Parameters) extends LoadForwardQueryIO {
332  // val sqIdx = Output(new SqPtr) // for debug, should not be used in pipeline for timing reasons
333  // sqIdxMask is calcuated in earlier stage for better timing
334  val sqIdxMask = Output(UInt(StoreQueueSize.W))
335
336  // dataInvalid: addr match, but data is not valid for now
337  val dataInvalidFast = Input(Bool()) // resp to load_s1
338  // val dataInvalid = Input(Bool()) // resp to load_s2
339  val dataInvalidSqIdx = Input(new SqPtr) // resp to load_s2, sqIdx
340  val addrInvalidSqIdx = Input(new SqPtr) // resp to load_s2, sqIdx
341}
342
343// Query load queue for ld-ld violation
344//
345// Req should be send in load_s1
346// Resp will be generated 1 cycle later
347//
348// Note that query req may be !ready, as dcache is releasing a block
349// If it happens, a replay from rs is needed.
350class LoadNukeQueryReq(implicit p: Parameters) extends XSBundle { // provide lqIdx
351  val uop = new DynInst
352  // mask: load's data mask.
353  val mask = UInt((VLEN/8).W)
354
355  // paddr: load's paddr.
356  val paddr      = UInt(PAddrBits.W)
357  // dataInvalid: load data is invalid.
358  val data_valid = Bool()
359}
360
361class LoadNukeQueryResp(implicit p: Parameters) extends XSBundle {
362  // rep_frm_fetch: ld-ld violation check success, replay from fetch.
363  val rep_frm_fetch = Bool()
364}
365
366class LoadNukeQueryIO(implicit p: Parameters) extends XSBundle {
367  val req    = Decoupled(new LoadNukeQueryReq)
368  val resp   = Flipped(Valid(new LoadNukeQueryResp))
369  val revoke = Output(Bool())
370}
371
372class StoreNukeQueryIO(implicit p: Parameters) extends XSBundle {
373  //  robIdx: Requestor's (a store instruction) rob index for match logic.
374  val robIdx = new RobPtr
375
376  //  paddr: requestor's (a store instruction) physical address for match logic.
377  val paddr  = UInt(PAddrBits.W)
378
379  //  mask: requestor's (a store instruction) data width mask for match logic.
380  val mask = UInt((VLEN/8).W)
381
382  // matchLine: if store is vector 128-bits, load unit need to compare 128-bits vaddr.
383  val matchLine = Bool()
384}
385
386class StoreMaBufToSqControlIO(implicit p: Parameters) extends XSBundle {
387  // from storeMisalignBuffer to storeQueue, control it's sbuffer write
388  val control = Output(new XSBundle {
389    // control sq to write-into sb
390    val writeSb = Bool()
391    val wdata = UInt(VLEN.W)
392    val wmask = UInt((VLEN / 8).W)
393    val paddr = UInt(PAddrBits.W)
394    val vaddr = UInt(VAddrBits.W)
395    val last  = Bool()
396    val hasException = Bool()
397    // remove this entry in sq
398    val removeSq = Bool()
399  })
400  // from storeQueue to storeMisalignBuffer, provide detail info of this store
401  val storeInfo = Input(new XSBundle {
402    val data = UInt(VLEN.W)
403    // is the data of the unaligned store ready at sq?
404    val dataReady = Bool()
405    // complete a data transfer from sq to sb
406    val completeSbTrans = Bool()
407  })
408}
409
410// Store byte valid mask write bundle
411//
412// Store byte valid mask write to SQ takes 2 cycles
413class StoreMaskBundle(implicit p: Parameters) extends XSBundle {
414  val sqIdx = new SqPtr
415  val mask = UInt((VLEN/8).W)
416}
417
418class LoadDataFromDcacheBundle(implicit p: Parameters) extends DCacheBundle {
419  // old dcache: optimize data sram read fanout
420  // val bankedDcacheData = Vec(DCacheBanks, UInt(64.W))
421  // val bank_oh = UInt(DCacheBanks.W)
422
423  // new dcache
424  val respDcacheData = UInt(VLEN.W)
425  val forwardMask = Vec(VLEN/8, Bool())
426  val forwardData = Vec(VLEN/8, UInt(8.W))
427  val uop = new DynInst // for data selection, only fwen and fuOpType are used
428  val addrOffset = UInt(4.W) // for data selection
429
430  // forward tilelink D channel
431  val forward_D = Bool()
432  val forwardData_D = Vec(VLEN/8, UInt(8.W))
433
434  // forward mshr data
435  val forward_mshr = Bool()
436  val forwardData_mshr = Vec(VLEN/8, UInt(8.W))
437
438  val forward_result_valid = Bool()
439
440  def mergeTLData(): UInt = {
441    // merge TL D or MSHR data at load s2
442    val dcache_data = respDcacheData
443    val use_D = forward_D && forward_result_valid
444    val use_mshr = forward_mshr && forward_result_valid
445    Mux(
446      use_D || use_mshr,
447      Mux(
448        use_D,
449        forwardData_D.asUInt,
450        forwardData_mshr.asUInt
451      ),
452      dcache_data
453    )
454  }
455
456  def mergeLsqFwdData(dcacheData: UInt): UInt = {
457    // merge dcache and lsq forward data at load s3
458    val rdataVec = VecInit((0 until VLEN / 8).map(j =>
459      Mux(forwardMask(j), forwardData(j), dcacheData(8*(j+1)-1, 8*j))
460    ))
461    rdataVec.asUInt
462  }
463}
464
465// Load writeback data from load queue (refill)
466class LoadDataFromLQBundle(implicit p: Parameters) extends XSBundle {
467  val lqData = UInt(64.W) // load queue has merged data
468  val uop = new DynInst // for data selection, only fwen and fuOpType are used
469  val addrOffset = UInt(3.W) // for data selection
470
471  def mergedData(): UInt = {
472    lqData
473  }
474}
475
476// Bundle for load / store wait waking up
477class MemWaitUpdateReq(implicit p: Parameters) extends XSBundle {
478  val robIdx = Vec(backendParams.StaExuCnt, ValidIO(new RobPtr))
479  val sqIdx = Vec(backendParams.StdCnt, ValidIO(new SqPtr))
480}
481
482object AddPipelineReg {
483  class PipelineRegModule[T <: Data](gen: T) extends Module {
484    val io = IO(new Bundle() {
485      val in = Flipped(DecoupledIO(gen.cloneType))
486      val out = DecoupledIO(gen.cloneType)
487      val isFlush = Input(Bool())
488    })
489
490    val valid = RegInit(false.B)
491    valid.suggestName("pipeline_reg_valid")
492    when (io.out.fire) { valid := false.B }
493    when (io.in.fire) { valid := true.B }
494    when (io.isFlush) { valid := false.B }
495
496    io.in.ready := !valid || io.out.ready
497    io.out.bits := RegEnable(io.in.bits, io.in.fire)
498    io.out.valid := valid //&& !isFlush
499  }
500
501  def apply[T <: Data]
502  (left: DecoupledIO[T], right: DecoupledIO[T], isFlush: Bool,
503   moduleName: Option[String] = None
504  ): Unit = {
505    val pipelineReg = Module(new PipelineRegModule[T](left.bits.cloneType))
506    if(moduleName.nonEmpty) pipelineReg.suggestName(moduleName.get)
507    pipelineReg.io.in <> left
508    right <> pipelineReg.io.out
509    pipelineReg.io.isFlush := isFlush
510  }
511}