xref: /XiangShan/src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala (revision 881e32f5b63c435bafbaf5dc1d792ffcc9ea103e)
1/***************************************************************************************
2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
3* Copyright (c) 2020-2021 Peng Cheng Laboratory
4*
5* XiangShan is licensed under Mulan PSL v2.
6* You can use this software according to the terms and conditions of the Mulan PSL v2.
7* You may obtain a copy of Mulan PSL v2 at:
8*          http://license.coscl.org.cn/MulanPSL2
9*
10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
13*
14* See the Mulan PSL v2 for more details.
15***************************************************************************************/
16
17package xiangshan.mem
18
19import org.chipsalliance.cde.config.Parameters
20import chisel3._
21import chisel3.util._
22import utils._
23import utility._
24import xiangshan.ExceptionNO._
25import xiangshan._
26import xiangshan.backend.Bundles.{DynInst, MemExuInput, MemExuOutput}
27import xiangshan.backend.fu.PMPRespBundle
28import xiangshan.backend.fu.FuConfig._
29import xiangshan.backend.fu.FuType
30import xiangshan.backend.ctrlblock.{DebugLsInfoBundle, LsTopdownInfo}
31import xiangshan.backend.rob.RobPtr
32import xiangshan.backend.ctrlblock.DebugLsInfoBundle
33import xiangshan.backend.fu.NewCSR._
34import xiangshan.backend.fu.util.SdtrigExt
35import xiangshan.cache._
36import xiangshan.cache.wpu.ReplayCarry
37import xiangshan.cache.mmu._
38import xiangshan.mem.mdp._
39
40class LoadToLsqReplayIO(implicit p: Parameters) extends XSBundle
41  with HasDCacheParameters
42  with HasTlbConst
43{
44  // mshr refill index
45  val mshr_id         = UInt(log2Up(cfg.nMissEntries).W)
46  // get full data from store queue and sbuffer
47  val full_fwd        = Bool()
48  // wait for data from store inst's store queue index
49  val data_inv_sq_idx = new SqPtr
50  // wait for address from store queue index
51  val addr_inv_sq_idx = new SqPtr
52  // replay carry
53  val rep_carry       = new ReplayCarry(nWays)
54  // data in last beat
55  val last_beat       = Bool()
56  // replay cause
57  val cause           = Vec(LoadReplayCauses.allCauses, Bool())
58  // performance debug information
59  val debug           = new PerfDebugInfo
60  // tlb hint
61  val tlb_id          = UInt(log2Up(loadfiltersize).W)
62  val tlb_full        = Bool()
63
64  // alias
65  def mem_amb       = cause(LoadReplayCauses.C_MA)
66  def tlb_miss      = cause(LoadReplayCauses.C_TM)
67  def fwd_fail      = cause(LoadReplayCauses.C_FF)
68  def dcache_rep    = cause(LoadReplayCauses.C_DR)
69  def dcache_miss   = cause(LoadReplayCauses.C_DM)
70  def wpu_fail      = cause(LoadReplayCauses.C_WF)
71  def bank_conflict = cause(LoadReplayCauses.C_BC)
72  def rar_nack      = cause(LoadReplayCauses.C_RAR)
73  def raw_nack      = cause(LoadReplayCauses.C_RAW)
74  def misalign_nack = cause(LoadReplayCauses.C_MF)
75  def nuke          = cause(LoadReplayCauses.C_NK)
76  def need_rep      = cause.asUInt.orR
77}
78
79
80class LoadToLsqIO(implicit p: Parameters) extends XSBundle {
81  // ldu -> lsq UncacheBuffer
82  val ldin            = DecoupledIO(new LqWriteBundle)
83  // uncache-mmio -> ldu
84  val uncache         = Flipped(DecoupledIO(new MemExuOutput))
85  val ld_raw_data     = Input(new LoadDataFromLQBundle)
86  // uncache-nc -> ldu
87  val nc_ldin = Flipped(DecoupledIO(new LsPipelineBundle))
88  // storequeue -> ldu
89  val forward         = new PipeLoadForwardQueryIO
90  // ldu -> lsq LQRAW
91  val stld_nuke_query = new LoadNukeQueryIO
92  // ldu -> lsq LQRAR
93  val ldld_nuke_query = new LoadNukeQueryIO
94}
95
96class LoadToLoadIO(implicit p: Parameters) extends XSBundle {
97  val valid      = Bool()
98  val data       = UInt(XLEN.W) // load to load fast path is limited to ld (64 bit) used as vaddr src1 only
99  val dly_ld_err = Bool()
100}
101
102class LoadUnitTriggerIO(implicit p: Parameters) extends XSBundle {
103  val tdata2      = Input(UInt(64.W))
104  val matchType   = Input(UInt(2.W))
105  val tEnable     = Input(Bool()) // timing is calculated before this
106  val addrHit     = Output(Bool())
107}
108
109class LoadUnit(implicit p: Parameters) extends XSModule
110  with HasLoadHelper
111  with HasPerfEvents
112  with HasDCacheParameters
113  with HasCircularQueuePtrHelper
114  with HasVLSUParameters
115  with SdtrigExt
116{
117  val io = IO(new Bundle() {
118    // control
119    val redirect      = Flipped(ValidIO(new Redirect))
120    val csrCtrl       = Flipped(new CustomCSRCtrlIO)
121
122    // int issue path
123    val ldin          = Flipped(Decoupled(new MemExuInput))
124    val ldout         = Decoupled(new MemExuOutput)
125
126    // vec issue path
127    val vecldin = Flipped(Decoupled(new VecPipeBundle))
128    val vecldout = Decoupled(new VecPipelineFeedbackIO(isVStore = false))
129
130    // misalignBuffer issue path
131    val misalign_ldin = Flipped(Decoupled(new LsPipelineBundle))
132    val misalign_ldout = Valid(new LqWriteBundle)
133
134    // data path
135    val tlb           = new TlbRequestIO(2)
136    val pmp           = Flipped(new PMPRespBundle()) // arrive same to tlb now
137    val dcache        = new DCacheLoadIO
138    val sbuffer       = new LoadForwardQueryIO
139    val ubuffer       = new LoadForwardQueryIO
140    val lsq           = new LoadToLsqIO
141    val tl_d_channel  = Input(new DcacheToLduForwardIO)
142    val forward_mshr  = Flipped(new LduToMissqueueForwardIO)
143   // val refill        = Flipped(ValidIO(new Refill))
144    val l2_hint       = Input(Valid(new L2ToL1Hint))
145    val tlb_hint      = Flipped(new TlbHintReq)
146    // fast wakeup
147    // TODO: implement vector fast wakeup
148    val fast_uop = ValidIO(new DynInst) // early wakeup signal generated in load_s1, send to RS in load_s2
149
150    // trigger
151    val fromCsrTrigger = Input(new CsrTriggerBundle)
152
153    // prefetch
154    val prefetch_train            = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to sms
155    val prefetch_train_l1         = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to stream & stride
156    // speculative for gated control
157    val s1_prefetch_spec = Output(Bool())
158    val s2_prefetch_spec = Output(Bool())
159
160    val prefetch_req              = Flipped(ValidIO(new L1PrefetchReq)) // hardware prefetch to l1 cache req
161    val canAcceptLowConfPrefetch  = Output(Bool())
162    val canAcceptHighConfPrefetch = Output(Bool())
163
164    // ifetchPrefetch
165    val ifetchPrefetch = ValidIO(new SoftIfetchPrefetchBundle)
166
167    // load to load fast path
168    val l2l_fwd_in    = Input(new LoadToLoadIO)
169    val l2l_fwd_out   = Output(new LoadToLoadIO)
170
171    val ld_fast_match    = Input(Bool())
172    val ld_fast_fuOpType = Input(UInt())
173    val ld_fast_imm      = Input(UInt(12.W))
174
175    // rs feedback
176    val wakeup = ValidIO(new DynInst)
177    val feedback_fast = ValidIO(new RSFeedback) // stage 2
178    val feedback_slow = ValidIO(new RSFeedback) // stage 3
179    val ldCancel = Output(new LoadCancelIO()) // use to cancel the uops waked by this load, and cancel load
180
181    // load ecc error
182    val s3_dly_ld_err = Output(Bool()) // Note that io.s3_dly_ld_err and io.lsq.s3_dly_ld_err is different
183
184    // schedule error query
185    val stld_nuke_query = Flipped(Vec(StorePipelineWidth, Valid(new StoreNukeQueryIO)))
186
187    // queue-based replay
188    val replay       = Flipped(Decoupled(new LsPipelineBundle))
189    val lq_rep_full  = Input(Bool())
190
191    // misc
192    val s2_ptr_chasing = Output(Bool()) // provide right pc for hw prefetch
193
194    // Load fast replay path
195    val fast_rep_in  = Flipped(Decoupled(new LqWriteBundle))
196    val fast_rep_out = Decoupled(new LqWriteBundle)
197
198    // to misalign buffer
199    val misalign_buf = Decoupled(new LqWriteBundle)
200
201    // Load RAR rollback
202    val rollback = Valid(new Redirect)
203
204    // perf
205    val debug_ls         = Output(new DebugLsInfoBundle)
206    val lsTopdownInfo    = Output(new LsTopdownInfo)
207    val correctMissTrain = Input(Bool())
208  })
209
210  val s1_ready, s2_ready, s3_ready = WireInit(false.B)
211
212  // Pipeline
213  // --------------------------------------------------------------------------------
214  // stage 0
215  // --------------------------------------------------------------------------------
216  // generate addr, use addr to query DCache and DTLB
217  val s0_valid         = Wire(Bool())
218  val s0_mmio_select   = Wire(Bool())
219  val s0_nc_select     = Wire(Bool())
220  val s0_misalign_select= Wire(Bool())
221  val s0_kill          = Wire(Bool())
222  val s0_can_go        = s1_ready
223  val s0_fire          = s0_valid && s0_can_go
224  val s0_mmio_fire     = s0_mmio_select && s0_can_go
225  val s0_nc_fire       = s0_nc_select && s0_can_go
226  val s0_out           = Wire(new LqWriteBundle)
227  val s0_tlb_valid     = Wire(Bool())
228  val s0_tlb_hlv       = Wire(Bool())
229  val s0_tlb_hlvx      = Wire(Bool())
230  val s0_tlb_vaddr     = Wire(UInt(VAddrBits.W))
231  val s0_tlb_fullva    = Wire(UInt(XLEN.W))
232  val s0_dcache_vaddr  = Wire(UInt(VAddrBits.W))
233  val s0_is128bit      = Wire(Bool())
234  val s0_misalign_wakeup_fire = s0_misalign_select && s0_can_go && io.misalign_ldin.bits.misalignNeedWakeUp
235
236  // flow source bundle
237  class FlowSource extends Bundle {
238    val vaddr         = UInt(VAddrBits.W)
239    val mask          = UInt((VLEN/8).W)
240    val uop           = new DynInst
241    val try_l2l       = Bool()
242    val has_rob_entry = Bool()
243    val rep_carry     = new ReplayCarry(nWays)
244    val mshrid        = UInt(log2Up(cfg.nMissEntries).W)
245    val isFirstIssue  = Bool()
246    val fast_rep      = Bool()
247    val ld_rep        = Bool()
248    val l2l_fwd       = Bool()
249    val prf           = Bool()
250    val prf_rd        = Bool()
251    val prf_wr        = Bool()
252    val prf_i         = Bool()
253    val sched_idx     = UInt(log2Up(LoadQueueReplaySize+1).W)
254    // Record the issue port idx of load issue queue. This signal is used by load cancel.
255    val deqPortIdx    = UInt(log2Ceil(LoadPipelineWidth).W)
256    val frm_mabuf     = Bool()
257    // vec only
258    val isvec         = Bool()
259    val is128bit      = Bool()
260    val uop_unit_stride_fof = Bool()
261    val reg_offset    = UInt(vOffsetBits.W)
262    val vecActive     = Bool() // 1: vector active element or scala mem operation, 0: vector not active element
263    val is_first_ele  = Bool()
264    // val flowPtr       = new VlflowPtr
265    val usSecondInv   = Bool()
266    val mbIndex       = UInt(vlmBindexBits.W)
267    val elemIdx       = UInt(elemIdxBits.W)
268    val elemIdxInsideVd = UInt(elemIdxBits.W)
269    val alignedType   = UInt(alignTypeBits.W)
270    val vecBaseVaddr  = UInt(VAddrBits.W)
271    //for Svpbmt NC
272    val isnc          = Bool()
273    val paddr         = UInt(PAddrBits.W)
274    val data          = UInt((VLEN+1).W)
275  }
276  val s0_sel_src = Wire(new FlowSource)
277
278  // load flow select/gen
279  // src 0: misalignBuffer load (io.misalign_ldin)
280  // src 1: super load replayed by LSQ (cache miss replay) (io.replay)
281  // src 2: fast load replay (io.fast_rep_in)
282  // src 3: mmio (io.lsq.uncache)
283  // src 4: nc (io.lsq.nc_ldin)
284  // src 5: load replayed by LSQ (io.replay)
285  // src 6: hardware prefetch from prefetchor (high confidence) (io.prefetch)
286  // NOTE: Now vec/int loads are sent from same RS
287  //       A vec load will be splited into multiple uops,
288  //       so as long as one uop is issued,
289  //       the other uops should have higher priority
290  // src 7: vec read from RS (io.vecldin)
291  // src 8: int read / software prefetch first issue from RS (io.in)
292  // src 9: load try pointchaising when no issued or replayed load (io.fastpath)
293  // src10: hardware prefetch from prefetchor (high confidence) (io.prefetch)
294  // priority: high to low
295  val s0_rep_stall           = io.ldin.valid && isAfter(io.replay.bits.uop.lqIdx, io.ldin.bits.uop.lqIdx) ||
296                               io.vecldin.valid && isAfter(io.replay.bits.uop.lqIdx, io.vecldin.bits.uop.lqIdx)
297  private val SRC_NUM = 11
298  private val Seq(
299    mab_idx, super_rep_idx, fast_rep_idx, mmio_idx, nc_idx, lsq_rep_idx,
300    high_pf_idx, vec_iss_idx, int_iss_idx, l2l_fwd_idx, low_pf_idx
301  ) = (0 until SRC_NUM).toSeq
302  // load flow source valid
303  val s0_src_valid_vec = WireInit(VecInit(Seq(
304    io.misalign_ldin.valid,
305    io.replay.valid && io.replay.bits.forward_tlDchannel,
306    io.fast_rep_in.valid,
307    io.lsq.uncache.valid,
308    io.lsq.nc_ldin.valid,
309    io.replay.valid && !io.replay.bits.forward_tlDchannel && !s0_rep_stall,
310    io.prefetch_req.valid && io.prefetch_req.bits.confidence > 0.U,
311    io.vecldin.valid,
312    io.ldin.valid, // int flow first issue or software prefetch
313    io.l2l_fwd_in.valid,
314    io.prefetch_req.valid && io.prefetch_req.bits.confidence === 0.U,
315  )))
316  // load flow source ready
317  val s0_src_ready_vec = Wire(Vec(SRC_NUM, Bool()))
318  s0_src_ready_vec(0) := true.B
319  for(i <- 1 until SRC_NUM){
320    s0_src_ready_vec(i) := !s0_src_valid_vec.take(i).reduce(_ || _)
321  }
322  // load flow source select (OH)
323  val s0_src_select_vec = WireInit(VecInit((0 until SRC_NUM).map{i => s0_src_valid_vec(i) && s0_src_ready_vec(i)}))
324  val s0_hw_prf_select = s0_src_select_vec(high_pf_idx) || s0_src_select_vec(low_pf_idx)
325
326  val s0_tlb_no_query = s0_hw_prf_select || s0_sel_src.prf_i ||
327    s0_src_select_vec(fast_rep_idx) || s0_src_select_vec(mmio_idx) ||
328    s0_src_select_vec(nc_idx)
329  s0_valid := !s0_kill && (s0_src_select_vec(nc_idx) || ((
330    s0_src_valid_vec(mab_idx) ||
331    s0_src_valid_vec(super_rep_idx) ||
332    s0_src_valid_vec(fast_rep_idx) ||
333    s0_src_valid_vec(lsq_rep_idx) ||
334    s0_src_valid_vec(high_pf_idx) ||
335    s0_src_valid_vec(vec_iss_idx) ||
336    s0_src_valid_vec(int_iss_idx) ||
337    s0_src_valid_vec(l2l_fwd_idx) ||
338    s0_src_valid_vec(low_pf_idx)
339  ) && !s0_src_select_vec(mmio_idx) && io.dcache.req.ready))
340
341  s0_mmio_select := s0_src_select_vec(mmio_idx) && !s0_kill
342  s0_nc_select := s0_src_select_vec(nc_idx) && !s0_kill
343  //judgment: is NC with data or not.
344  //If true, it's from `io.lsq.nc_ldin` or `io.fast_rep_in`
345  val s0_nc_with_data = s0_sel_src.isnc && !s0_kill
346  s0_misalign_select := s0_src_select_vec(mab_idx) && !s0_kill
347
348   // if is hardware prefetch or fast replay, don't send valid to tlb
349  s0_tlb_valid := (
350    s0_src_valid_vec(mab_idx) ||
351    s0_src_valid_vec(super_rep_idx) ||
352    s0_src_valid_vec(lsq_rep_idx) ||
353    s0_src_valid_vec(vec_iss_idx) ||
354    s0_src_valid_vec(int_iss_idx) ||
355    s0_src_valid_vec(l2l_fwd_idx)
356  ) && io.dcache.req.ready
357
358  // which is S0's out is ready and dcache is ready
359  val s0_try_ptr_chasing      = s0_src_select_vec(l2l_fwd_idx)
360  val s0_do_try_ptr_chasing   = s0_try_ptr_chasing && s0_can_go && io.dcache.req.ready
361  val s0_ptr_chasing_vaddr    = io.l2l_fwd_in.data(5, 0) +& io.ld_fast_imm(5, 0)
362  val s0_ptr_chasing_canceled = WireInit(false.B)
363  s0_kill := s0_ptr_chasing_canceled
364
365  // prefetch related ctrl signal
366  io.canAcceptLowConfPrefetch  := s0_src_ready_vec(low_pf_idx) && io.dcache.req.ready
367  io.canAcceptHighConfPrefetch := s0_src_ready_vec(high_pf_idx) && io.dcache.req.ready
368
369  // query DTLB
370  io.tlb.req.valid                   := s0_tlb_valid
371  io.tlb.req.bits.cmd                := Mux(s0_sel_src.prf,
372                                         Mux(s0_sel_src.prf_wr, TlbCmd.write, TlbCmd.read),
373                                         TlbCmd.read
374                                       )
375  io.tlb.req.bits.isPrefetch         := s0_sel_src.prf
376  io.tlb.req.bits.vaddr              := s0_tlb_vaddr
377  io.tlb.req.bits.fullva             := s0_tlb_fullva
378  io.tlb.req.bits.checkfullva        := s0_src_select_vec(vec_iss_idx) || s0_src_select_vec(int_iss_idx)
379  io.tlb.req.bits.hyperinst          := s0_tlb_hlv
380  io.tlb.req.bits.hlvx               := s0_tlb_hlvx
381  io.tlb.req.bits.size               := Mux(s0_sel_src.isvec, s0_sel_src.alignedType(2,0), LSUOpType.size(s0_sel_src.uop.fuOpType))
382  io.tlb.req.bits.kill               := s0_kill || s0_tlb_no_query // if does not need to be translated, kill it
383  io.tlb.req.bits.memidx.is_ld       := true.B
384  io.tlb.req.bits.memidx.is_st       := false.B
385  io.tlb.req.bits.memidx.idx         := s0_sel_src.uop.lqIdx.value
386  io.tlb.req.bits.debug.robIdx       := s0_sel_src.uop.robIdx
387  io.tlb.req.bits.no_translate       := s0_tlb_no_query  // hardware prefetch and fast replay does not need to be translated, need this signal for pmp check
388  io.tlb.req.bits.debug.pc           := s0_sel_src.uop.pc
389  io.tlb.req.bits.debug.isFirstIssue := s0_sel_src.isFirstIssue
390
391  // query DCache
392  io.dcache.req.valid             := s0_valid && !s0_sel_src.prf_i && !s0_nc_with_data
393  io.dcache.req.bits.cmd          := Mux(s0_sel_src.prf_rd,
394                                      MemoryOpConstants.M_PFR,
395                                      Mux(s0_sel_src.prf_wr, MemoryOpConstants.M_PFW, MemoryOpConstants.M_XRD)
396                                    )
397  io.dcache.req.bits.vaddr        := s0_dcache_vaddr
398  io.dcache.req.bits.vaddr_dup    := s0_dcache_vaddr
399  io.dcache.req.bits.mask         := s0_sel_src.mask
400  io.dcache.req.bits.data         := DontCare
401  io.dcache.req.bits.isFirstIssue := s0_sel_src.isFirstIssue
402  io.dcache.req.bits.instrtype    := Mux(s0_sel_src.prf, DCACHE_PREFETCH_SOURCE.U, LOAD_SOURCE.U)
403  io.dcache.req.bits.debug_robIdx := s0_sel_src.uop.robIdx.value
404  io.dcache.req.bits.replayCarry  := s0_sel_src.rep_carry
405  io.dcache.req.bits.id           := DontCare // TODO: update cache meta
406  io.dcache.req.bits.lqIdx        := s0_sel_src.uop.lqIdx
407  io.dcache.pf_source             := Mux(s0_hw_prf_select, io.prefetch_req.bits.pf_source.value, L1_HW_PREFETCH_NULL)
408  io.dcache.is128Req              := s0_is128bit
409
410  // load flow priority mux
411  def fromNullSource(): FlowSource = {
412    val out = WireInit(0.U.asTypeOf(new FlowSource))
413    out
414  }
415
416  def fromMisAlignBufferSource(src: LsPipelineBundle): FlowSource = {
417    val out = WireInit(0.U.asTypeOf(new FlowSource))
418    out.vaddr         := src.vaddr
419    out.mask          := src.mask
420    out.uop           := src.uop
421    out.try_l2l       := false.B
422    out.has_rob_entry := false.B
423    out.rep_carry     := src.replayCarry
424    out.mshrid        := src.mshrid
425    out.frm_mabuf     := true.B
426    out.isFirstIssue  := false.B
427    out.fast_rep      := false.B
428    out.ld_rep        := false.B
429    out.l2l_fwd       := false.B
430    out.prf           := false.B
431    out.prf_rd        := false.B
432    out.prf_wr        := false.B
433    out.sched_idx     := src.schedIndex
434    out.isvec         := src.isvec
435    out.is128bit      := src.is128bit
436    out.vecActive     := true.B
437    out
438  }
439
440  def fromFastReplaySource(src: LqWriteBundle): FlowSource = {
441    val out = WireInit(0.U.asTypeOf(new FlowSource))
442    out.vaddr         := src.vaddr
443    out.paddr         := src.paddr
444    out.mask          := src.mask
445    out.uop           := src.uop
446    out.try_l2l       := false.B
447    out.has_rob_entry := src.hasROBEntry
448    out.rep_carry     := src.rep_info.rep_carry
449    out.mshrid        := src.rep_info.mshr_id
450    out.frm_mabuf     := src.isFrmMisAlignBuf
451    out.isFirstIssue  := false.B
452    out.fast_rep      := true.B
453    out.ld_rep        := src.isLoadReplay
454    out.l2l_fwd       := false.B
455    out.prf           := LSUOpType.isPrefetch(src.uop.fuOpType) && !src.isvec
456    out.prf_rd        := src.uop.fuOpType === LSUOpType.prefetch_r
457    out.prf_wr        := src.uop.fuOpType === LSUOpType.prefetch_w
458    out.prf_i         := false.B
459    out.sched_idx     := src.schedIndex
460    out.isvec         := src.isvec
461    out.is128bit      := src.is128bit
462    out.uop_unit_stride_fof := src.uop_unit_stride_fof
463    out.reg_offset    := src.reg_offset
464    out.vecActive     := src.vecActive
465    out.is_first_ele  := src.is_first_ele
466    out.usSecondInv   := src.usSecondInv
467    out.mbIndex       := src.mbIndex
468    out.elemIdx       := src.elemIdx
469    out.elemIdxInsideVd := src.elemIdxInsideVd
470    out.alignedType   := src.alignedType
471    out.isnc          := src.nc
472    out.data          := src.data
473    out
474  }
475
476  // TODO: implement vector mmio
477  def fromMmioSource(src: MemExuOutput) = {
478    val out = WireInit(0.U.asTypeOf(new FlowSource))
479    out.mask          := 0.U
480    out.uop           := src.uop
481    out.try_l2l       := false.B
482    out.has_rob_entry := false.B
483    out.rep_carry     := 0.U.asTypeOf(out.rep_carry)
484    out.mshrid        := 0.U
485    out.frm_mabuf     := false.B
486    out.isFirstIssue  := false.B
487    out.fast_rep      := false.B
488    out.ld_rep        := false.B
489    out.l2l_fwd       := false.B
490    out.prf           := false.B
491    out.prf_rd        := false.B
492    out.prf_wr        := false.B
493    out.prf_i         := false.B
494    out.sched_idx     := 0.U
495    out.vecActive     := true.B
496    out
497  }
498
499  def fromNcSource(src: LsPipelineBundle): FlowSource = {
500    val out = WireInit(0.U.asTypeOf(new FlowSource))
501    out.vaddr := src.vaddr
502    out.paddr := src.paddr
503    out.mask := genVWmask(src.vaddr, src.uop.fuOpType(1,0))
504    out.uop := src.uop
505    out.has_rob_entry := true.B
506    out.sched_idx := src.schedIndex
507    out.isvec := src.isvec
508    out.is128bit := src.is128bit
509    out.vecActive := src.vecActive
510    out.isnc := true.B
511    out.data := src.data
512    out
513  }
514
515  def fromNormalReplaySource(src: LsPipelineBundle): FlowSource = {
516    val out = WireInit(0.U.asTypeOf(new FlowSource))
517    out.mask          := Mux(src.isvec, src.mask, genVWmask(src.vaddr, src.uop.fuOpType(1, 0)))
518    out.uop           := src.uop
519    out.try_l2l       := false.B
520    out.has_rob_entry := true.B
521    out.rep_carry     := src.replayCarry
522    out.mshrid        := src.mshrid
523    out.frm_mabuf     := false.B
524    out.isFirstIssue  := false.B
525    out.fast_rep      := false.B
526    out.ld_rep        := true.B
527    out.l2l_fwd       := false.B
528    out.prf           := LSUOpType.isPrefetch(src.uop.fuOpType) && !src.isvec
529    out.prf_rd        := src.uop.fuOpType === LSUOpType.prefetch_r
530    out.prf_wr        := src.uop.fuOpType === LSUOpType.prefetch_w
531    out.prf_i         := false.B
532    out.sched_idx     := src.schedIndex
533    out.isvec         := src.isvec
534    out.is128bit      := src.is128bit
535    out.uop_unit_stride_fof := src.uop_unit_stride_fof
536    out.reg_offset    := src.reg_offset
537    out.vecActive     := src.vecActive
538    out.is_first_ele  := src.is_first_ele
539    out.usSecondInv   := src.usSecondInv
540    out.mbIndex       := src.mbIndex
541    out.elemIdx       := src.elemIdx
542    out.elemIdxInsideVd := src.elemIdxInsideVd
543    out.alignedType   := src.alignedType
544    out
545  }
546
547  // TODO: implement vector prefetch
548  def fromPrefetchSource(src: L1PrefetchReq): FlowSource = {
549    val out = WireInit(0.U.asTypeOf(new FlowSource))
550    out.mask          := 0.U
551    out.uop           := DontCare
552    out.try_l2l       := false.B
553    out.has_rob_entry := false.B
554    out.rep_carry     := 0.U.asTypeOf(out.rep_carry)
555    out.mshrid        := 0.U
556    out.frm_mabuf     := false.B
557    out.isFirstIssue  := false.B
558    out.fast_rep      := false.B
559    out.ld_rep        := false.B
560    out.l2l_fwd       := false.B
561    out.prf           := true.B
562    out.prf_rd        := !src.is_store
563    out.prf_wr        := src.is_store
564    out.prf_i         := false.B
565    out.sched_idx     := 0.U
566    out
567  }
568
569  def fromVecIssueSource(src: VecPipeBundle): FlowSource = {
570    val out = WireInit(0.U.asTypeOf(new FlowSource))
571    out.mask          := src.mask
572    out.uop           := src.uop
573    out.try_l2l       := false.B
574    out.has_rob_entry := true.B
575    // TODO: VLSU, implement replay carry
576    out.rep_carry     := 0.U.asTypeOf(out.rep_carry)
577    out.mshrid        := 0.U
578    out.frm_mabuf     := false.B
579    // TODO: VLSU, implement first issue
580//    out.isFirstIssue  := src.isFirstIssue
581    out.fast_rep      := false.B
582    out.ld_rep        := false.B
583    out.l2l_fwd       := false.B
584    out.prf           := false.B
585    out.prf_rd        := false.B
586    out.prf_wr        := false.B
587    out.prf_i         := false.B
588    out.sched_idx     := 0.U
589    // Vector load interface
590    out.isvec               := true.B
591    // vector loads only access a single element at a time, so 128-bit path is not used for now
592    out.is128bit            := is128Bit(src.alignedType)
593    out.uop_unit_stride_fof := src.uop_unit_stride_fof
594    // out.rob_idx_valid       := src.rob_idx_valid
595    // out.inner_idx           := src.inner_idx
596    // out.rob_idx             := src.rob_idx
597    out.reg_offset          := src.reg_offset
598    // out.offset              := src.offset
599    out.vecActive           := src.vecActive
600    out.is_first_ele        := src.is_first_ele
601    // out.flowPtr             := src.flowPtr
602    out.usSecondInv         := src.usSecondInv
603    out.mbIndex             := src.mBIndex
604    out.elemIdx             := src.elemIdx
605    out.elemIdxInsideVd     := src.elemIdxInsideVd
606    out.vecBaseVaddr        := src.basevaddr
607    out.alignedType         := src.alignedType
608    out
609  }
610
611  def fromIntIssueSource(src: MemExuInput): FlowSource = {
612    val out = WireInit(0.U.asTypeOf(new FlowSource))
613    val addr           = io.ldin.bits.src(0) + SignExt(io.ldin.bits.uop.imm(11, 0), VAddrBits)
614    out.mask          := genVWmask(addr, src.uop.fuOpType(1,0))
615    out.uop           := src.uop
616    out.try_l2l       := false.B
617    out.has_rob_entry := true.B
618    out.rep_carry     := 0.U.asTypeOf(out.rep_carry)
619    out.mshrid        := 0.U
620    out.frm_mabuf     := false.B
621    out.isFirstIssue  := true.B
622    out.fast_rep      := false.B
623    out.ld_rep        := false.B
624    out.l2l_fwd       := false.B
625    out.prf           := LSUOpType.isPrefetch(src.uop.fuOpType)
626    out.prf_rd        := src.uop.fuOpType === LSUOpType.prefetch_r
627    out.prf_wr        := src.uop.fuOpType === LSUOpType.prefetch_w
628    out.prf_i         := src.uop.fuOpType === LSUOpType.prefetch_i
629    out.sched_idx     := 0.U
630    out.vecActive     := true.B // true for scala load
631    out
632  }
633
634  // TODO: implement vector l2l
635  def fromLoadToLoadSource(src: LoadToLoadIO): FlowSource = {
636    val out = WireInit(0.U.asTypeOf(new FlowSource))
637    out.mask               := genVWmask(0.U, LSUOpType.ld)
638    // When there's no valid instruction from RS and LSQ, we try the load-to-load forwarding.
639    // Assume the pointer chasing is always ld.
640    out.uop.fuOpType       := LSUOpType.ld
641    out.try_l2l            := true.B
642    // we dont care out.isFirstIssue and out.rsIdx and s0_sqIdx in S0 when trying pointchasing
643    // because these signals will be updated in S1
644    out.has_rob_entry      := false.B
645    out.mshrid             := 0.U
646    out.frm_mabuf          := false.B
647    out.rep_carry          := 0.U.asTypeOf(out.rep_carry)
648    out.isFirstIssue       := true.B
649    out.fast_rep           := false.B
650    out.ld_rep             := false.B
651    out.l2l_fwd            := true.B
652    out.prf                := false.B
653    out.prf_rd             := false.B
654    out.prf_wr             := false.B
655    out.prf_i              := false.B
656    out.sched_idx          := 0.U
657    out
658  }
659
660  // set default
661  val s0_src_selector = WireInit(s0_src_valid_vec)
662  if (!EnableLoadToLoadForward) { s0_src_selector(l2l_fwd_idx) := false.B }
663  val s0_src_format = Seq(
664    fromMisAlignBufferSource(io.misalign_ldin.bits),
665    fromNormalReplaySource(io.replay.bits),
666    fromFastReplaySource(io.fast_rep_in.bits),
667    fromMmioSource(io.lsq.uncache.bits),
668    fromNcSource(io.lsq.nc_ldin.bits),
669    fromNormalReplaySource(io.replay.bits),
670    fromPrefetchSource(io.prefetch_req.bits),
671    fromVecIssueSource(io.vecldin.bits),
672    fromIntIssueSource(io.ldin.bits),
673    (if (EnableLoadToLoadForward) fromLoadToLoadSource(io.l2l_fwd_in) else fromNullSource()),
674    fromPrefetchSource(io.prefetch_req.bits)
675  )
676  s0_sel_src := ParallelPriorityMux(s0_src_selector, s0_src_format)
677
678  // fast replay and hardware prefetch don't need to query tlb
679  val int_issue_vaddr = io.ldin.bits.src(0) + SignExt(io.ldin.bits.uop.imm(11, 0), VAddrBits)
680  val int_vec_vaddr = Mux(s0_src_valid_vec(vec_iss_idx), io.vecldin.bits.vaddr(VAddrBits - 1, 0), int_issue_vaddr)
681  s0_tlb_vaddr := Mux(
682    s0_src_valid_vec(mab_idx),
683    io.misalign_ldin.bits.vaddr,
684    Mux(
685      s0_src_valid_vec(super_rep_idx) || s0_src_valid_vec(lsq_rep_idx),
686      io.replay.bits.vaddr,
687      int_vec_vaddr
688    )
689  )
690  s0_dcache_vaddr := Mux(
691    s0_src_select_vec(fast_rep_idx), io.fast_rep_in.bits.vaddr,
692    Mux(s0_hw_prf_select, io.prefetch_req.bits.getVaddr(),
693    Mux(s0_src_select_vec(nc_idx), io.lsq.nc_ldin.bits.vaddr, // not for dcache access, but for address alignment check
694    s0_tlb_vaddr))
695  )
696
697  val s0_alignType = Mux(s0_sel_src.isvec, s0_sel_src.alignedType(1,0), s0_sel_src.uop.fuOpType(1, 0))
698
699  val s0_addr_aligned = LookupTree(s0_alignType, List(
700    "b00".U   -> true.B,                   //b
701    "b01".U   -> (s0_dcache_vaddr(0)    === 0.U), //h
702    "b10".U   -> (s0_dcache_vaddr(1, 0) === 0.U), //w
703    "b11".U   -> (s0_dcache_vaddr(2, 0) === 0.U)  //d
704  ))
705  // address align check
706  XSError(s0_sel_src.isvec && s0_dcache_vaddr(3, 0) =/= 0.U && s0_sel_src.alignedType(2), "unit-stride 128 bit element is not aligned!")
707
708  val s0_check_vaddr_low = s0_dcache_vaddr(4, 0)
709  val s0_check_vaddr_Up_low = LookupTree(s0_alignType, List(
710    "b00".U -> 0.U,
711    "b01".U -> 1.U,
712    "b10".U -> 3.U,
713    "b11".U -> 7.U
714  )) + s0_check_vaddr_low
715  //TODO vec?
716  val s0_rs_cross16Bytes = s0_check_vaddr_Up_low(4) =/= s0_check_vaddr_low(4)
717  val s0_misalignWith16Byte = !s0_rs_cross16Bytes && !s0_addr_aligned && !s0_hw_prf_select
718  val s0_misalignNeedWakeUp = s0_sel_src.frm_mabuf && io.misalign_ldin.bits.misalignNeedWakeUp
719  val s0_finalSplit = s0_sel_src.frm_mabuf && io.misalign_ldin.bits.isFinalSplit
720  s0_is128bit := s0_sel_src.is128bit || s0_misalignWith16Byte
721
722  // only first issue of int / vec load intructions need to check full vaddr
723  s0_tlb_fullva := Mux(s0_src_valid_vec(mab_idx),
724    io.misalign_ldin.bits.fullva,
725    Mux(s0_src_select_vec(vec_iss_idx),
726      io.vecldin.bits.vaddr,
727      Mux(
728        s0_src_select_vec(int_iss_idx),
729        io.ldin.bits.src(0) + SignExt(io.ldin.bits.uop.imm(11, 0), XLEN),
730        s0_dcache_vaddr
731      )
732    )
733  )
734
735  s0_tlb_hlv := Mux(
736    s0_src_valid_vec(mab_idx),
737    LSUOpType.isHlv(io.misalign_ldin.bits.uop.fuOpType),
738    Mux(
739      s0_src_valid_vec(super_rep_idx) || s0_src_valid_vec(lsq_rep_idx),
740      LSUOpType.isHlv(io.replay.bits.uop.fuOpType),
741      Mux(
742        s0_src_valid_vec(int_iss_idx),
743        LSUOpType.isHlv(io.ldin.bits.uop.fuOpType),
744        false.B
745      )
746    )
747  )
748  s0_tlb_hlvx := Mux(
749    s0_src_valid_vec(mab_idx),
750    LSUOpType.isHlvx(io.misalign_ldin.bits.uop.fuOpType),
751    Mux(
752      s0_src_valid_vec(super_rep_idx) || s0_src_valid_vec(lsq_rep_idx),
753      LSUOpType.isHlvx(io.replay.bits.uop.fuOpType),
754      Mux(
755        s0_src_valid_vec(int_iss_idx),
756        LSUOpType.isHlvx(io.ldin.bits.uop.fuOpType),
757        false.B
758      )
759    )
760  )
761
762  // accept load flow if dcache ready (tlb is always ready)
763  // TODO: prefetch need writeback to loadQueueFlag
764  s0_out               := DontCare
765  s0_out.vaddr         := Mux(s0_nc_with_data, s0_sel_src.vaddr, s0_dcache_vaddr)
766  s0_out.fullva        := s0_tlb_fullva
767  s0_out.mask          := s0_sel_src.mask
768  s0_out.uop           := s0_sel_src.uop
769  s0_out.isFirstIssue  := s0_sel_src.isFirstIssue
770  s0_out.hasROBEntry   := s0_sel_src.has_rob_entry
771  s0_out.isPrefetch    := s0_sel_src.prf
772  s0_out.isHWPrefetch  := s0_hw_prf_select
773  s0_out.isFastReplay  := s0_sel_src.fast_rep
774  s0_out.isLoadReplay  := s0_sel_src.ld_rep
775  s0_out.isFastPath    := s0_sel_src.l2l_fwd
776  s0_out.mshrid        := s0_sel_src.mshrid
777  s0_out.isvec           := s0_sel_src.isvec
778  s0_out.is128bit        := s0_is128bit
779  s0_out.isFrmMisAlignBuf    := s0_sel_src.frm_mabuf
780  s0_out.uop_unit_stride_fof := s0_sel_src.uop_unit_stride_fof
781  s0_out.paddr         :=
782    Mux(s0_src_valid_vec(nc_idx), io.lsq.nc_ldin.bits.paddr,
783    Mux(s0_src_valid_vec(fast_rep_idx), io.fast_rep_in.bits.paddr,
784    Mux(s0_src_select_vec(int_iss_idx) && s0_sel_src.prf_i, 0.U,
785    io.prefetch_req.bits.paddr))) // only for nc, fast_rep, prefetch
786  s0_out.tlbNoQuery    := s0_tlb_no_query
787  // s0_out.rob_idx_valid   := s0_rob_idx_valid
788  // s0_out.inner_idx       := s0_inner_idx
789  // s0_out.rob_idx         := s0_rob_idx
790  s0_out.reg_offset      := s0_sel_src.reg_offset
791  // s0_out.offset          := s0_offset
792  s0_out.vecActive             := s0_sel_src.vecActive
793  s0_out.usSecondInv    := s0_sel_src.usSecondInv
794  s0_out.is_first_ele   := s0_sel_src.is_first_ele
795  s0_out.elemIdx        := s0_sel_src.elemIdx
796  s0_out.elemIdxInsideVd := s0_sel_src.elemIdxInsideVd
797  s0_out.alignedType    := s0_sel_src.alignedType
798  s0_out.mbIndex        := s0_sel_src.mbIndex
799  s0_out.vecBaseVaddr   := s0_sel_src.vecBaseVaddr
800  // s0_out.flowPtr         := s0_sel_src.flowPtr
801  s0_out.uop.exceptionVec(loadAddrMisaligned) := (!s0_addr_aligned || s0_sel_src.uop.exceptionVec(loadAddrMisaligned)) && s0_sel_src.vecActive && !s0_misalignWith16Byte
802  s0_out.isMisalign := (!s0_addr_aligned || s0_sel_src.uop.exceptionVec(loadAddrMisaligned)) && s0_sel_src.vecActive
803  s0_out.forward_tlDchannel := s0_src_select_vec(super_rep_idx)
804  when(io.tlb.req.valid && s0_sel_src.isFirstIssue) {
805    s0_out.uop.debugInfo.tlbFirstReqTime := GTimer()
806  }.otherwise{
807    s0_out.uop.debugInfo.tlbFirstReqTime := s0_sel_src.uop.debugInfo.tlbFirstReqTime
808  }
809  s0_out.schedIndex     := s0_sel_src.sched_idx
810  //for Svpbmt Nc
811  s0_out.nc := s0_sel_src.isnc
812  s0_out.data := s0_sel_src.data
813  s0_out.misalignWith16Byte    := s0_misalignWith16Byte
814  s0_out.misalignNeedWakeUp := s0_misalignNeedWakeUp
815  s0_out.isFinalSplit := s0_finalSplit
816
817  // load fast replay
818  io.fast_rep_in.ready := (s0_can_go && io.dcache.req.ready && s0_src_ready_vec(fast_rep_idx))
819
820  // mmio
821  io.lsq.uncache.ready := s0_mmio_fire
822  io.lsq.nc_ldin.ready := s0_src_ready_vec(nc_idx) && s0_can_go
823
824  // load flow source ready
825  // cache missed load has highest priority
826  // always accept cache missed load flow from load replay queue
827  io.replay.ready := (s0_can_go && io.dcache.req.ready && (s0_src_ready_vec(lsq_rep_idx) && !s0_rep_stall || s0_src_select_vec(super_rep_idx)))
828
829  // accept load flow from rs when:
830  // 1) there is no lsq-replayed load
831  // 2) there is no fast replayed load
832  // 3) there is no high confidence prefetch request
833  io.vecldin.ready := s0_can_go && io.dcache.req.ready && s0_src_ready_vec(vec_iss_idx)
834  io.ldin.ready := s0_can_go && io.dcache.req.ready && s0_src_ready_vec(int_iss_idx)
835  io.misalign_ldin.ready := s0_can_go && io.dcache.req.ready && s0_src_ready_vec(mab_idx)
836
837  // for hw prefetch load flow feedback, to be added later
838  // io.prefetch_in.ready := s0_hw_prf_select
839
840  // dcache replacement extra info
841  // TODO: should prefetch load update replacement?
842  io.dcache.replacementUpdated := Mux(s0_src_select_vec(lsq_rep_idx) || s0_src_select_vec(super_rep_idx), io.replay.bits.replacementUpdated, false.B)
843
844  // load wakeup
845  // TODO: vector load wakeup? frm_mabuf wakeup?
846  val s0_wakeup_selector = Seq(
847    s0_misalign_wakeup_fire,
848    s0_src_valid_vec(super_rep_idx),
849    s0_src_valid_vec(fast_rep_idx),
850    s0_mmio_fire,
851    s0_nc_fire,
852    s0_src_valid_vec(lsq_rep_idx),
853    s0_src_valid_vec(int_iss_idx)
854  )
855  val s0_wakeup_format = Seq(
856    io.misalign_ldin.bits.uop,
857    io.replay.bits.uop,
858    io.fast_rep_in.bits.uop,
859    io.lsq.uncache.bits.uop,
860    io.lsq.nc_ldin.bits.uop,
861    io.replay.bits.uop,
862    io.ldin.bits.uop,
863  )
864  val s0_wakeup_uop = ParallelPriorityMux(s0_wakeup_selector, s0_wakeup_format)
865  io.wakeup.valid := s0_fire && !s0_sel_src.isvec && !s0_sel_src.frm_mabuf && (
866    s0_src_valid_vec(super_rep_idx) ||
867    s0_src_valid_vec(fast_rep_idx) ||
868    s0_src_valid_vec(lsq_rep_idx) ||
869    (s0_src_valid_vec(int_iss_idx) && !s0_sel_src.prf &&
870    !s0_src_valid_vec(vec_iss_idx) && !s0_src_valid_vec(high_pf_idx))
871  ) || s0_mmio_fire || s0_nc_fire || s0_misalign_wakeup_fire
872  io.wakeup.bits := s0_wakeup_uop
873
874  // prefetch.i(Zicbop)
875  io.ifetchPrefetch.valid := RegNext(s0_src_select_vec(int_iss_idx) && s0_sel_src.prf_i)
876  io.ifetchPrefetch.bits.vaddr := RegEnable(s0_out.vaddr, 0.U, s0_src_select_vec(int_iss_idx) && s0_sel_src.prf_i)
877
878  XSDebug(io.dcache.req.fire,
879    p"[DCACHE LOAD REQ] pc ${Hexadecimal(s0_sel_src.uop.pc)}, vaddr ${Hexadecimal(s0_dcache_vaddr)}\n"
880  )
881  XSDebug(s0_valid,
882    p"S0: pc ${Hexadecimal(s0_out.uop.pc)}, lId ${Hexadecimal(s0_out.uop.lqIdx.asUInt)}, " +
883    p"vaddr ${Hexadecimal(s0_out.vaddr)}, mask ${Hexadecimal(s0_out.mask)}\n")
884
885  // Pipeline
886  // --------------------------------------------------------------------------------
887  // stage 1
888  // --------------------------------------------------------------------------------
889  // TLB resp (send paddr to dcache)
890  val s1_valid      = RegInit(false.B)
891  val s1_in         = Wire(new LqWriteBundle)
892  val s1_out        = Wire(new LqWriteBundle)
893  val s1_kill       = Wire(Bool())
894  val s1_can_go     = s2_ready
895  val s1_fire       = s1_valid && !s1_kill && s1_can_go
896  val s1_vecActive        = RegEnable(s0_out.vecActive, true.B, s0_fire)
897  val s1_nc_with_data = RegNext(s0_nc_with_data)
898
899  s1_ready := !s1_valid || s1_kill || s2_ready
900  when (s0_fire) { s1_valid := true.B }
901  .elsewhen (s1_fire) { s1_valid := false.B }
902  .elsewhen (s1_kill) { s1_valid := false.B }
903  s1_in   := RegEnable(s0_out, s0_fire)
904
905  val s1_fast_rep_dly_kill = RegEnable(io.fast_rep_in.bits.lateKill, io.fast_rep_in.valid) && s1_in.isFastReplay
906  val s1_fast_rep_dly_err =  RegEnable(io.fast_rep_in.bits.delayedLoadError, io.fast_rep_in.valid) && s1_in.isFastReplay
907  val s1_l2l_fwd_dly_err  = RegEnable(io.l2l_fwd_in.dly_ld_err, io.l2l_fwd_in.valid) && s1_in.isFastPath
908  val s1_dly_err          = s1_fast_rep_dly_err || s1_l2l_fwd_dly_err
909  val s1_vaddr_hi         = Wire(UInt())
910  val s1_vaddr_lo         = Wire(UInt())
911  val s1_vaddr            = Wire(UInt())
912  val s1_paddr_dup_lsu    = Wire(UInt())
913  val s1_gpaddr_dup_lsu   = Wire(UInt())
914  val s1_paddr_dup_dcache = Wire(UInt())
915  val s1_exception        = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, LduCfg).asUInt.orR   // af & pf exception were modified below.
916  val s1_tlb_miss         = io.tlb.resp.bits.miss && io.tlb.resp.valid && s1_valid
917  val s1_tlb_fast_miss    = io.tlb.resp.bits.fastMiss && io.tlb.resp.valid && s1_valid
918  val s1_pbmt             = Mux(!s1_tlb_miss, io.tlb.resp.bits.pbmt.head, 0.U(Pbmt.width.W))
919  val s1_nc               = s1_in.nc
920  val s1_prf              = s1_in.isPrefetch
921  val s1_hw_prf           = s1_in.isHWPrefetch
922  val s1_sw_prf           = s1_prf && !s1_hw_prf
923  val s1_tlb_memidx       = io.tlb.resp.bits.memidx
924
925  s1_vaddr_hi         := s1_in.vaddr(VAddrBits - 1, 6)
926  s1_vaddr_lo         := s1_in.vaddr(5, 0)
927  s1_vaddr            := Cat(s1_vaddr_hi, s1_vaddr_lo)
928  s1_paddr_dup_lsu    := Mux(s1_in.tlbNoQuery, s1_in.paddr, io.tlb.resp.bits.paddr(0))
929  s1_paddr_dup_dcache := Mux(s1_in.tlbNoQuery, s1_in.paddr, io.tlb.resp.bits.paddr(1))
930  s1_gpaddr_dup_lsu   := Mux(s1_in.isFastReplay, s1_in.paddr, io.tlb.resp.bits.gpaddr(0))
931
932  when (s1_tlb_memidx.is_ld && io.tlb.resp.valid && !s1_tlb_miss && s1_tlb_memidx.idx === s1_in.uop.lqIdx.value) {
933    // printf("load idx = %d\n", s1_tlb_memidx.idx)
934    s1_out.uop.debugInfo.tlbRespTime := GTimer()
935  }
936
937  io.tlb.req_kill   := s1_kill || s1_dly_err
938  io.tlb.req.bits.pmp_addr := s1_in.paddr
939  io.tlb.resp.ready := true.B
940
941  io.dcache.s1_paddr_dup_lsu    <> s1_paddr_dup_lsu
942  io.dcache.s1_paddr_dup_dcache <> s1_paddr_dup_dcache
943  io.dcache.s1_kill             := s1_kill || s1_dly_err || s1_tlb_miss || s1_exception
944  io.dcache.s1_kill_data_read   := s1_kill || s1_dly_err || s1_tlb_fast_miss
945
946  // store to load forwarding
947  io.sbuffer.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_dly_err || s1_prf)
948  io.sbuffer.vaddr := s1_vaddr
949  io.sbuffer.paddr := s1_paddr_dup_lsu
950  io.sbuffer.uop   := s1_in.uop
951  io.sbuffer.sqIdx := s1_in.uop.sqIdx
952  io.sbuffer.mask  := s1_in.mask
953  io.sbuffer.pc    := s1_in.uop.pc // FIXME: remove it
954
955  io.ubuffer.valid := s1_valid && s1_nc_with_data && !(s1_exception || s1_tlb_miss || s1_kill || s1_dly_err || s1_prf)
956  io.ubuffer.vaddr := s1_vaddr
957  io.ubuffer.paddr := s1_paddr_dup_lsu
958  io.ubuffer.uop   := s1_in.uop
959  io.ubuffer.sqIdx := s1_in.uop.sqIdx
960  io.ubuffer.mask  := s1_in.mask
961  io.ubuffer.pc    := s1_in.uop.pc // FIXME: remove it
962
963  io.lsq.forward.valid     := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_dly_err || s1_prf)
964  io.lsq.forward.vaddr     := s1_vaddr
965  io.lsq.forward.paddr     := s1_paddr_dup_lsu
966  io.lsq.forward.uop       := s1_in.uop
967  io.lsq.forward.sqIdx     := s1_in.uop.sqIdx
968  io.lsq.forward.sqIdxMask := 0.U
969  io.lsq.forward.mask      := s1_in.mask
970  io.lsq.forward.pc        := s1_in.uop.pc // FIXME: remove it
971
972  // st-ld violation query
973    // if store unit is 128-bits memory access, need match 128-bit
974  private val s1_isMatch128 = io.stld_nuke_query.map(x => (x.bits.matchLine || ((s1_in.isvec || s1_in.misalignWith16Byte) && s1_in.is128bit)))
975  val s1_nuke_paddr_match = VecInit((0 until StorePipelineWidth).zip(s1_isMatch128).map{case (w, s) => {Mux(s,
976    s1_paddr_dup_lsu(PAddrBits-1, 4) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 4),
977    s1_paddr_dup_lsu(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3))}})
978  val s1_nuke = VecInit((0 until StorePipelineWidth).map(w => {
979                       io.stld_nuke_query(w).valid && // query valid
980                       isAfter(s1_in.uop.robIdx, io.stld_nuke_query(w).bits.robIdx) && // older store
981                       s1_nuke_paddr_match(w) && // paddr match
982                       (s1_in.mask & io.stld_nuke_query(w).bits.mask).orR // data mask contain
983                      })).asUInt.orR && !s1_tlb_miss
984
985  s1_out                   := s1_in
986  s1_out.vaddr             := s1_vaddr
987  s1_out.fullva            := io.tlb.resp.bits.fullva
988  s1_out.vaNeedExt         := io.tlb.resp.bits.excp(0).vaNeedExt
989  s1_out.isHyper           := io.tlb.resp.bits.excp(0).isHyper
990  s1_out.paddr             := s1_paddr_dup_lsu
991  s1_out.gpaddr            := s1_gpaddr_dup_lsu
992  s1_out.isForVSnonLeafPTE := io.tlb.resp.bits.isForVSnonLeafPTE
993  s1_out.tlbMiss           := s1_tlb_miss
994  s1_out.ptwBack           := io.tlb.resp.bits.ptwBack
995  s1_out.rep_info.debug    := s1_in.uop.debugInfo
996  s1_out.rep_info.nuke     := s1_nuke && !s1_sw_prf
997  s1_out.delayedLoadError  := s1_dly_err
998  s1_out.nc := s1_nc || Pbmt.isNC(s1_pbmt)
999  s1_out.mmio := Pbmt.isIO(s1_pbmt)
1000
1001  when (!s1_dly_err) {
1002    // current ori test will cause the case of ldest == 0, below will be modifeid in the future.
1003    // af & pf exception were modified
1004    // if is tlbNoQuery request, don't trigger exception from tlb resp
1005    s1_out.uop.exceptionVec(loadPageFault)   := io.tlb.resp.bits.excp(0).pf.ld && s1_vecActive && !s1_tlb_miss && !s1_in.tlbNoQuery
1006    s1_out.uop.exceptionVec(loadGuestPageFault)   := io.tlb.resp.bits.excp(0).gpf.ld && !s1_tlb_miss && !s1_in.tlbNoQuery
1007    s1_out.uop.exceptionVec(loadAccessFault) := io.tlb.resp.bits.excp(0).af.ld && s1_vecActive && !s1_tlb_miss && !s1_in.tlbNoQuery
1008    when (RegNext(io.tlb.req.bits.checkfullva) &&
1009      (s1_out.uop.exceptionVec(loadPageFault) ||
1010        s1_out.uop.exceptionVec(loadGuestPageFault) ||
1011        s1_out.uop.exceptionVec(loadAccessFault))) {
1012      s1_out.uop.exceptionVec(loadAddrMisaligned) := false.B
1013      s1_out.isMisalign := false.B
1014    }
1015  } .otherwise {
1016    s1_out.uop.exceptionVec(loadPageFault)      := false.B
1017    s1_out.uop.exceptionVec(loadGuestPageFault) := false.B
1018    s1_out.uop.exceptionVec(loadAddrMisaligned) := false.B
1019    s1_out.isMisalign := false.B
1020    s1_out.uop.exceptionVec(loadAccessFault)    := s1_dly_err && s1_vecActive
1021  }
1022
1023  // pointer chasing
1024  val s1_try_ptr_chasing       = GatedValidRegNext(s0_do_try_ptr_chasing, false.B)
1025  val s1_ptr_chasing_vaddr     = RegEnable(s0_ptr_chasing_vaddr, s0_do_try_ptr_chasing)
1026  val s1_fu_op_type_not_ld     = WireInit(false.B)
1027  val s1_not_fast_match        = WireInit(false.B)
1028  val s1_addr_mismatch         = WireInit(false.B)
1029  val s1_addr_misaligned       = WireInit(false.B)
1030  val s1_fast_mismatch         = WireInit(false.B)
1031  val s1_ptr_chasing_canceled  = WireInit(false.B)
1032  val s1_cancel_ptr_chasing    = WireInit(false.B)
1033
1034  val s1_redirect_reg = Wire(Valid(new Redirect))
1035  s1_redirect_reg.bits := RegEnable(io.redirect.bits, io.redirect.valid)
1036  s1_redirect_reg.valid := GatedValidRegNext(io.redirect.valid)
1037
1038  s1_kill := s1_fast_rep_dly_kill ||
1039    s1_cancel_ptr_chasing ||
1040    s1_in.uop.robIdx.needFlush(io.redirect) ||
1041    (s1_in.uop.robIdx.needFlush(s1_redirect_reg) && !GatedValidRegNext(s0_try_ptr_chasing)) ||
1042    RegEnable(s0_kill, false.B, io.ldin.valid ||
1043      io.vecldin.valid || io.replay.valid ||
1044      io.l2l_fwd_in.valid || io.fast_rep_in.valid ||
1045      io.misalign_ldin.valid || io.lsq.nc_ldin.valid
1046    )
1047
1048  if (EnableLoadToLoadForward) {
1049    // Sometimes, we need to cancel the load-load forwarding.
1050    // These can be put at S0 if timing is bad at S1.
1051    // Case 0: CACHE_SET(base + offset) != CACHE_SET(base) (lowest 6-bit addition has an overflow)
1052    s1_addr_mismatch     := s1_ptr_chasing_vaddr(6) ||
1053                             RegEnable(io.ld_fast_imm(11, 6).orR, s0_do_try_ptr_chasing)
1054    // Case 1: the address is not 64-bit aligned or the fuOpType is not LD
1055    s1_addr_misaligned := s1_ptr_chasing_vaddr(2, 0).orR
1056    s1_fu_op_type_not_ld := io.ldin.bits.uop.fuOpType =/= LSUOpType.ld
1057    // Case 2: this load-load uop is cancelled
1058    s1_ptr_chasing_canceled := !io.ldin.valid
1059    // Case 3: fast mismatch
1060    s1_fast_mismatch := RegEnable(!io.ld_fast_match, s0_do_try_ptr_chasing)
1061
1062    when (s1_try_ptr_chasing) {
1063      s1_cancel_ptr_chasing := s1_addr_mismatch ||
1064                               s1_addr_misaligned ||
1065                               s1_fu_op_type_not_ld ||
1066                               s1_ptr_chasing_canceled ||
1067                               s1_fast_mismatch
1068
1069      s1_in.uop           := io.ldin.bits.uop
1070      s1_in.isFirstIssue  := io.ldin.bits.isFirstIssue
1071      s1_vaddr_lo         := s1_ptr_chasing_vaddr(5, 0)
1072      s1_paddr_dup_lsu    := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo)
1073      s1_paddr_dup_dcache := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo)
1074
1075      // recored tlb time when get the data to ensure the correctness of the latency calculation (although it should not record in here, because it does not use tlb)
1076      s1_in.uop.debugInfo.tlbFirstReqTime := GTimer()
1077      s1_in.uop.debugInfo.tlbRespTime     := GTimer()
1078    }
1079    when (!s1_cancel_ptr_chasing) {
1080      s0_ptr_chasing_canceled := s1_try_ptr_chasing &&
1081        !io.replay.fire && !io.fast_rep_in.fire &&
1082        !(s0_src_valid_vec(high_pf_idx) && io.canAcceptHighConfPrefetch) &&
1083        !io.misalign_ldin.fire &&
1084        !io.lsq.nc_ldin.valid
1085      when (s1_try_ptr_chasing) {
1086        io.ldin.ready := true.B
1087      }
1088    }
1089  }
1090
1091  // pre-calcuate sqIdx mask in s0, then send it to lsq in s1 for forwarding
1092  val s1_sqIdx_mask = RegEnable(UIntToMask(s0_out.uop.sqIdx.value, StoreQueueSize), s0_fire)
1093  // to enable load-load, sqIdxMask must be calculated based on ldin.uop
1094  // If the timing here is not OK, load-load forwarding has to be disabled.
1095  // Or we calculate sqIdxMask at RS??
1096  io.lsq.forward.sqIdxMask := s1_sqIdx_mask
1097  if (EnableLoadToLoadForward) {
1098    when (s1_try_ptr_chasing) {
1099      io.lsq.forward.sqIdxMask := UIntToMask(io.ldin.bits.uop.sqIdx.value, StoreQueueSize)
1100    }
1101  }
1102
1103  io.forward_mshr.valid  := s1_valid && s1_out.forward_tlDchannel
1104  io.forward_mshr.mshrid := s1_out.mshrid
1105  io.forward_mshr.paddr  := s1_out.paddr
1106
1107  val loadTrigger = Module(new MemTrigger(MemType.LOAD))
1108  loadTrigger.io.fromCsrTrigger.tdataVec             := io.fromCsrTrigger.tdataVec
1109  loadTrigger.io.fromCsrTrigger.tEnableVec           := io.fromCsrTrigger.tEnableVec
1110  loadTrigger.io.fromCsrTrigger.triggerCanRaiseBpExp := io.fromCsrTrigger.triggerCanRaiseBpExp
1111  loadTrigger.io.fromCsrTrigger.debugMode            := io.fromCsrTrigger.debugMode
1112  loadTrigger.io.fromLoadStore.vaddr                 := s1_vaddr
1113  loadTrigger.io.fromLoadStore.isVectorUnitStride    := s1_in.isvec && s1_in.is128bit
1114  loadTrigger.io.fromLoadStore.mask                  := s1_in.mask
1115
1116  val s1_trigger_action = loadTrigger.io.toLoadStore.triggerAction
1117  val s1_trigger_debug_mode = TriggerAction.isDmode(s1_trigger_action)
1118  val s1_trigger_breakpoint = TriggerAction.isExp(s1_trigger_action)
1119  s1_out.uop.trigger                  := s1_trigger_action
1120  s1_out.uop.exceptionVec(breakPoint) := s1_trigger_breakpoint
1121  s1_out.vecVaddrOffset := Mux(
1122    s1_trigger_debug_mode || s1_trigger_breakpoint,
1123    loadTrigger.io.toLoadStore.triggerVaddr - s1_in.vecBaseVaddr,
1124    s1_in.vaddr + genVFirstUnmask(s1_in.mask).asUInt - s1_in.vecBaseVaddr
1125  )
1126  s1_out.vecTriggerMask := Mux(s1_trigger_debug_mode || s1_trigger_breakpoint, loadTrigger.io.toLoadStore.triggerMask, 0.U)
1127
1128  XSDebug(s1_valid,
1129    p"S1: pc ${Hexadecimal(s1_out.uop.pc)}, lId ${Hexadecimal(s1_out.uop.lqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " +
1130    p"paddr ${Hexadecimal(s1_out.paddr)}, mmio ${s1_out.mmio}\n")
1131
1132  // Pipeline
1133  // --------------------------------------------------------------------------------
1134  // stage 2
1135  // --------------------------------------------------------------------------------
1136  // s2: DCache resp
1137  val s2_valid  = RegInit(false.B)
1138  val s2_in     = Wire(new LqWriteBundle)
1139  val s2_out    = Wire(new LqWriteBundle)
1140  val s2_kill   = Wire(Bool())
1141  val s2_can_go = s3_ready
1142  val s2_fire   = s2_valid && !s2_kill && s2_can_go
1143  val s2_vecActive = RegEnable(s1_out.vecActive, true.B, s1_fire)
1144  val s2_isvec  = RegEnable(s1_out.isvec, false.B, s1_fire)
1145  val s2_data_select  = genRdataOH(s2_out.uop)
1146  val s2_data_select_by_offset = genDataSelectByOffset(s2_out.paddr(3, 0))
1147  val s2_frm_mabuf = s2_in.isFrmMisAlignBuf
1148  val s2_pbmt = RegEnable(s1_pbmt, s1_fire)
1149  val s2_trigger_debug_mode = RegEnable(s1_trigger_debug_mode, false.B, s1_fire)
1150  val s2_nc_with_data = RegNext(s1_nc_with_data)
1151  val s2_mmio_req = Wire(Valid(new MemExuOutput))
1152  s2_mmio_req.valid := RegNextN(io.lsq.uncache.fire, 2, Some(false.B))
1153  s2_mmio_req.bits  := RegNextN(io.lsq.uncache.bits, 2)
1154
1155  s2_kill := s2_in.uop.robIdx.needFlush(io.redirect)
1156  s2_ready := !s2_valid || s2_kill || s3_ready
1157  when (s1_fire) { s2_valid := true.B }
1158  .elsewhen (s2_fire) { s2_valid := false.B }
1159  .elsewhen (s2_kill) { s2_valid := false.B }
1160  s2_in := RegEnable(s1_out, s1_fire)
1161
1162  val s2_pmp = WireInit(io.pmp)
1163  val s2_isMisalign = WireInit(s2_in.isMisalign)
1164
1165  val s2_prf    = s2_in.isPrefetch
1166  val s2_hw_prf = s2_in.isHWPrefetch
1167  val s2_exception_vec = WireInit(s2_in.uop.exceptionVec)
1168  val s2_un_misalign_exception =  s2_vecActive &&
1169                                  (s2_trigger_debug_mode || ExceptionNO.selectByFuAndUnSelect(s2_exception_vec, LduCfg, Seq(loadAddrMisaligned)).asUInt.orR)
1170  val s2_check_mmio = !s2_prf && !s2_in.tlbMiss && Mux(Pbmt.isUncache(s2_pbmt), s2_in.mmio, s2_pmp.mmio) && !s2_un_misalign_exception
1171  // exception that may cause load addr to be invalid / illegal
1172  // if such exception happen, that inst and its exception info
1173  // will be force writebacked to rob
1174  val s2_actually_uncache = Pbmt.isPMA(s2_pbmt) && s2_pmp.mmio || s2_in.nc || s2_in.mmio
1175  val s2_memBackTypeMM = !s2_pmp.mmio
1176  when (!s2_in.delayedLoadError) {
1177    s2_exception_vec(loadAccessFault) := s2_vecActive && (
1178      s2_in.uop.exceptionVec(loadAccessFault) ||
1179      s2_pmp.ld ||
1180      (s2_isvec || s2_frm_mabuf) && s2_actually_uncache && !s2_prf && !s2_in.tlbMiss ||
1181      io.dcache.resp.bits.tag_error && GatedValidRegNext(io.csrCtrl.cache_error_enable)
1182    )
1183  }
1184
1185  // soft prefetch will not trigger any exception (but ecc error interrupt may
1186  // be triggered)
1187  val s2_tlb_unrelated_exceps = s2_in.uop.exceptionVec(loadAddrMisaligned) ||
1188                                s2_in.uop.exceptionVec(breakPoint)
1189  when (!s2_in.delayedLoadError && (s2_prf || s2_in.tlbMiss && !s2_tlb_unrelated_exceps)) {
1190    s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType)
1191    s2_isMisalign := false.B
1192  }
1193  val s2_exception = s2_vecActive &&
1194                    (s2_trigger_debug_mode || ExceptionNO.selectByFu(s2_exception_vec, LduCfg).asUInt.orR)
1195  val s2_mis_align = s2_valid && GatedValidRegNext(io.csrCtrl.hd_misalign_ld_enable) &&
1196                     s2_out.isMisalign && !s2_in.misalignWith16Byte && !s2_exception_vec(breakPoint) && !s2_trigger_debug_mode && !s2_check_mmio
1197  val (s2_fwd_frm_d_chan, s2_fwd_data_frm_d_chan, s2_d_corrupt) = io.tl_d_channel.forward(s1_valid && s1_out.forward_tlDchannel, s1_out.mshrid, s1_out.paddr)
1198  val (s2_fwd_data_valid, s2_fwd_frm_mshr, s2_fwd_data_frm_mshr, s2_mshr_corrupt) = io.forward_mshr.forward()
1199  val s2_fwd_frm_d_chan_or_mshr = s2_fwd_data_valid && (s2_fwd_frm_d_chan || s2_fwd_frm_mshr)
1200
1201  // writeback access fault caused by ecc error / bus error
1202  // * ecc data error is slow to generate, so we will not use it until load stage 3
1203  // * in load stage 3, an extra signal io.load_error will be used to
1204  // * if pbmt =/= 0, mmio is up to pbmt; otherwise, it's up to pmp
1205  val s2_mmio = !s2_prf &&
1206    !s2_exception && !s2_in.tlbMiss &&
1207    Mux(Pbmt.isUncache(s2_pbmt), s2_in.mmio, s2_pmp.mmio)
1208  val s2_uncache = !s2_prf && !s2_exception && !s2_in.tlbMiss && s2_actually_uncache
1209
1210  val s2_full_fwd      = Wire(Bool())
1211  val s2_mem_amb       = s2_in.uop.storeSetHit &&
1212                         io.lsq.forward.addrInvalid && RegNext(io.lsq.forward.valid)
1213
1214  val s2_tlb_miss      = s2_in.tlbMiss
1215  val s2_fwd_fail      = io.lsq.forward.dataInvalid && RegNext(io.lsq.forward.valid)
1216  val s2_dcache_miss   = io.dcache.resp.bits.miss &&
1217                         !s2_fwd_frm_d_chan_or_mshr &&
1218                         !s2_full_fwd && !s2_in.nc
1219
1220  val s2_mq_nack       = io.dcache.s2_mq_nack &&
1221                         !s2_fwd_frm_d_chan_or_mshr &&
1222                         !s2_full_fwd && !s2_in.nc
1223
1224  val s2_bank_conflict = io.dcache.s2_bank_conflict &&
1225                         !s2_fwd_frm_d_chan_or_mshr &&
1226                         !s2_full_fwd && !s2_in.nc
1227
1228  val s2_wpu_pred_fail = io.dcache.s2_wpu_pred_fail &&
1229                        !s2_fwd_frm_d_chan_or_mshr &&
1230                        !s2_full_fwd && !s2_in.nc
1231
1232  val s2_rar_nack      = io.lsq.ldld_nuke_query.req.valid &&
1233                         !io.lsq.ldld_nuke_query.req.ready
1234
1235  val s2_raw_nack      = io.lsq.stld_nuke_query.req.valid &&
1236                         !io.lsq.stld_nuke_query.req.ready
1237  // st-ld violation query
1238  //  NeedFastRecovery Valid when
1239  //  1. Fast recovery query request Valid.
1240  //  2. Load instruction is younger than requestors(store instructions).
1241  //  3. Physical address match.
1242  //  4. Data contains.
1243  private val s2_isMatch128 = io.stld_nuke_query.map(x => (x.bits.matchLine || ((s2_in.isvec || s2_in.misalignWith16Byte) && s2_in.is128bit)))
1244  val s2_nuke_paddr_match = VecInit((0 until StorePipelineWidth).zip(s2_isMatch128).map{case (w, s) => {Mux(s,
1245    s2_in.paddr(PAddrBits-1, 4) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 4),
1246    s2_in.paddr(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3))}})
1247  val s2_nuke          = VecInit((0 until StorePipelineWidth).map(w => {
1248                          io.stld_nuke_query(w).valid && // query valid
1249                          isAfter(s2_in.uop.robIdx, io.stld_nuke_query(w).bits.robIdx) && // older store
1250                          s2_nuke_paddr_match(w) && // paddr match
1251                          (s2_in.mask & io.stld_nuke_query(w).bits.mask).orR // data mask contain
1252                        })).asUInt.orR && !s2_tlb_miss || s2_in.rep_info.nuke
1253
1254  val s2_cache_handled   = io.dcache.resp.bits.handled
1255
1256  //if it is NC with data, it should handle the replayed situation.
1257  //else s2_uncache will enter uncache buffer.
1258  val s2_troublem        = !s2_exception &&
1259                           (!s2_uncache || s2_nc_with_data) &&
1260                           !s2_prf &&
1261                           !s2_in.delayedLoadError
1262
1263  io.dcache.resp.ready  := true.B
1264  val s2_dcache_should_resp = !(s2_in.tlbMiss || s2_exception || s2_in.delayedLoadError || s2_uncache || s2_prf)
1265  assert(!(s2_valid && (s2_dcache_should_resp && !io.dcache.resp.valid)), "DCache response got lost")
1266
1267  // fast replay require
1268  val s2_dcache_fast_rep = (s2_mq_nack || !s2_dcache_miss && (s2_bank_conflict || s2_wpu_pred_fail))
1269  val s2_nuke_fast_rep   = !s2_mq_nack &&
1270                           !s2_dcache_miss &&
1271                           !s2_bank_conflict &&
1272                           !s2_wpu_pred_fail &&
1273                           s2_nuke
1274
1275  val s2_fast_rep = !s2_in.isFastReplay &&
1276                    !s2_mem_amb &&
1277                    !s2_tlb_miss &&
1278                    !s2_fwd_fail &&
1279                    (s2_dcache_fast_rep || s2_nuke_fast_rep) &&
1280                    s2_troublem &&
1281                    !s2_in.misalignNeedWakeUp
1282
1283  // need allocate new entry
1284  val s2_can_query = !s2_mem_amb &&
1285                     !s2_tlb_miss &&
1286                     !s2_fwd_fail &&
1287                     !s2_frm_mabuf &&
1288                     !s2_fast_rep &&
1289                     s2_troublem
1290
1291  val s2_data_fwded = s2_dcache_miss && s2_full_fwd
1292
1293  // For misaligned, we will keep the misaligned exception at S2 and before.
1294  // Here a judgement is made as to whether a misaligned exception needs to actually be generated.
1295  // We will generate misaligned exceptions at mmio.
1296  val s2_real_exceptionVec = WireInit(s2_exception_vec)
1297  s2_real_exceptionVec(loadAddrMisaligned) := s2_out.isMisalign && s2_check_mmio
1298  s2_real_exceptionVec(loadAccessFault) := s2_exception_vec(loadAccessFault) ||
1299    s2_fwd_frm_d_chan && s2_d_corrupt ||
1300    s2_fwd_frm_mshr && s2_mshr_corrupt
1301  val s2_real_exception = s2_vecActive &&
1302    (s2_trigger_debug_mode || ExceptionNO.selectByFu(s2_real_exceptionVec, LduCfg).asUInt.orR)
1303
1304  val s2_fwd_vp_match_invalid = io.lsq.forward.matchInvalid || io.sbuffer.matchInvalid || io.ubuffer.matchInvalid
1305  val s2_vp_match_fail = s2_fwd_vp_match_invalid && s2_troublem
1306  val s2_safe_wakeup = !s2_out.rep_info.need_rep && !s2_mmio && (!s2_in.nc || s2_nc_with_data) && !s2_mis_align && !s2_real_exception || s2_in.misalignNeedWakeUp // don't need to replay and is not a mmio\misalign no data
1307  val s2_safe_writeback = s2_real_exception || s2_safe_wakeup || s2_vp_match_fail || s2_in.misalignNeedWakeUp
1308
1309  // ld-ld violation require
1310  io.lsq.ldld_nuke_query.req.valid           := s2_valid && s2_can_query
1311  io.lsq.ldld_nuke_query.req.bits.uop        := s2_in.uop
1312  io.lsq.ldld_nuke_query.req.bits.mask       := s2_in.mask
1313  io.lsq.ldld_nuke_query.req.bits.paddr      := s2_in.paddr
1314  io.lsq.ldld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid || s2_nc_with_data, true.B, !s2_dcache_miss)
1315  io.lsq.ldld_nuke_query.req.bits.is_nc := s2_nc_with_data
1316
1317  // st-ld violation require
1318  io.lsq.stld_nuke_query.req.valid           := s2_valid && s2_can_query
1319  io.lsq.stld_nuke_query.req.bits.uop        := s2_in.uop
1320  io.lsq.stld_nuke_query.req.bits.mask       := s2_in.mask
1321  io.lsq.stld_nuke_query.req.bits.paddr      := s2_in.paddr
1322  io.lsq.stld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid || s2_nc_with_data, true.B, !s2_dcache_miss)
1323  io.lsq.stld_nuke_query.req.bits.is_nc := s2_nc_with_data
1324
1325  // merge forward result
1326  // lsq has higher priority than sbuffer
1327  val s2_fwd_mask = Wire(Vec((VLEN/8), Bool()))
1328  val s2_fwd_data = Wire(Vec((VLEN/8), UInt(8.W)))
1329  s2_full_fwd := ((~s2_fwd_mask.asUInt).asUInt & s2_in.mask) === 0.U && !io.lsq.forward.dataInvalid
1330  // generate XLEN/8 Muxs
1331  for (i <- 0 until VLEN / 8) {
1332    s2_fwd_mask(i) := io.lsq.forward.forwardMask(i) || io.sbuffer.forwardMask(i) || io.ubuffer.forwardMask(i)
1333    s2_fwd_data(i) :=
1334      Mux(io.lsq.forward.forwardMask(i), io.lsq.forward.forwardData(i),
1335      Mux(s2_nc_with_data, io.ubuffer.forwardData(i),
1336      io.sbuffer.forwardData(i)))
1337  }
1338
1339  XSDebug(s2_fire, "[FWD LOAD RESP] pc %x fwd %x(%b) + %x(%b)\n",
1340    s2_in.uop.pc,
1341    io.lsq.forward.forwardData.asUInt, io.lsq.forward.forwardMask.asUInt,
1342    s2_in.forwardData.asUInt, s2_in.forwardMask.asUInt
1343  )
1344
1345  //
1346  s2_out                     := s2_in
1347  s2_out.uop.fpWen           := s2_in.uop.fpWen
1348  s2_out.nc                  := s2_in.nc
1349  s2_out.mmio                := s2_mmio
1350  s2_out.memBackTypeMM       := s2_memBackTypeMM
1351  s2_out.isMisalign          := s2_isMisalign
1352  s2_out.uop.flushPipe       := false.B
1353  s2_out.uop.exceptionVec    := s2_real_exceptionVec
1354  s2_out.forwardMask         := s2_fwd_mask
1355  s2_out.forwardData         := s2_fwd_data
1356  s2_out.handledByMSHR       := s2_cache_handled
1357  s2_out.miss                := s2_dcache_miss && s2_troublem
1358  s2_out.feedbacked          := io.feedback_fast.valid
1359  s2_out.uop.vpu.vstart      := Mux(s2_in.isLoadReplay || s2_in.isFastReplay, s2_in.uop.vpu.vstart, s2_in.vecVaddrOffset >> s2_in.uop.vpu.veew)
1360
1361  // Generate replay signal caused by:
1362  // * st-ld violation check
1363  // * tlb miss
1364  // * dcache replay
1365  // * forward data invalid
1366  // * dcache miss
1367  s2_out.rep_info.mem_amb         := s2_mem_amb && s2_troublem
1368  s2_out.rep_info.tlb_miss        := s2_tlb_miss && s2_troublem
1369  s2_out.rep_info.fwd_fail        := s2_fwd_fail && s2_troublem
1370  s2_out.rep_info.dcache_rep      := s2_mq_nack && s2_troublem
1371  s2_out.rep_info.dcache_miss     := s2_dcache_miss && s2_troublem
1372  s2_out.rep_info.bank_conflict   := s2_bank_conflict && s2_troublem
1373  s2_out.rep_info.wpu_fail        := s2_wpu_pred_fail && s2_troublem
1374  s2_out.rep_info.rar_nack        := s2_rar_nack && s2_troublem
1375  s2_out.rep_info.raw_nack        := s2_raw_nack && s2_troublem
1376  s2_out.rep_info.nuke            := s2_nuke && s2_troublem
1377  s2_out.rep_info.full_fwd        := s2_data_fwded
1378  s2_out.rep_info.data_inv_sq_idx := io.lsq.forward.dataInvalidSqIdx
1379  s2_out.rep_info.addr_inv_sq_idx := io.lsq.forward.addrInvalidSqIdx
1380  s2_out.rep_info.rep_carry       := io.dcache.resp.bits.replayCarry
1381  s2_out.rep_info.mshr_id         := io.dcache.resp.bits.mshr_id
1382  s2_out.rep_info.last_beat       := s2_in.paddr(log2Up(refillBytes))
1383  s2_out.rep_info.debug           := s2_in.uop.debugInfo
1384  s2_out.rep_info.tlb_id          := io.tlb_hint.id
1385  s2_out.rep_info.tlb_full        := io.tlb_hint.full
1386
1387  // if forward fail, replay this inst from fetch
1388  val debug_fwd_fail_rep = s2_fwd_fail && !s2_troublem && !s2_in.tlbMiss
1389  // if ld-ld violation is detected, replay from this inst from fetch
1390  val debug_ldld_nuke_rep = false.B // s2_ldld_violation && !s2_mmio && !s2_is_prefetch && !s2_in.tlbMiss
1391
1392  // to be removed
1393  io.feedback_fast.valid                 := false.B
1394  io.feedback_fast.bits.hit              := false.B
1395  io.feedback_fast.bits.flushState       := s2_in.ptwBack
1396  io.feedback_fast.bits.robIdx           := s2_in.uop.robIdx
1397  io.feedback_fast.bits.sqIdx            := s2_in.uop.sqIdx
1398  io.feedback_fast.bits.lqIdx            := s2_in.uop.lqIdx
1399  io.feedback_fast.bits.sourceType       := RSFeedbackType.lrqFull
1400  io.feedback_fast.bits.dataInvalidSqIdx := DontCare
1401
1402  io.ldCancel.ld1Cancel := false.B
1403
1404  // fast wakeup
1405  val s1_fast_uop_valid = WireInit(false.B)
1406  s1_fast_uop_valid :=
1407    !io.dcache.s1_disable_fast_wakeup &&
1408    s1_valid &&
1409    !s1_kill &&
1410    !io.tlb.resp.bits.miss &&
1411    !io.lsq.forward.dataInvalidFast
1412  io.fast_uop.valid := GatedValidRegNext(s1_fast_uop_valid) && (s2_valid && !s2_out.rep_info.need_rep && !s2_uncache && !(s2_prf && !s2_hw_prf)) && !s2_isvec && !s2_frm_mabuf
1413  io.fast_uop.bits := RegEnable(s1_out.uop, s1_fast_uop_valid)
1414
1415  //
1416  io.s2_ptr_chasing                    := RegEnable(s1_try_ptr_chasing && !s1_cancel_ptr_chasing, false.B, s1_fire)
1417
1418  // RegNext prefetch train for better timing
1419  // ** Now, prefetch train is valid at load s3 **
1420  val s2_prefetch_train_valid = WireInit(false.B)
1421  s2_prefetch_train_valid              := s2_valid && !s2_actually_uncache && (!s2_in.tlbMiss || s2_hw_prf)
1422  io.prefetch_train.valid              := GatedValidRegNext(s2_prefetch_train_valid)
1423  io.prefetch_train.bits.fromLsPipelineBundle(s2_in, latch = true, enable = s2_prefetch_train_valid)
1424  io.prefetch_train.bits.miss          := RegEnable(io.dcache.resp.bits.miss, s2_prefetch_train_valid) // TODO: use trace with bank conflict?
1425  io.prefetch_train.bits.meta_prefetch := RegEnable(io.dcache.resp.bits.meta_prefetch, s2_prefetch_train_valid)
1426  io.prefetch_train.bits.meta_access   := RegEnable(io.dcache.resp.bits.meta_access, s2_prefetch_train_valid)
1427  io.prefetch_train.bits.isFinalSplit      := false.B
1428  io.prefetch_train.bits.misalignWith16Byte := false.B
1429  io.prefetch_train.bits.misalignNeedWakeUp := false.B
1430  io.prefetch_train.bits.updateAddrValid := false.B
1431  io.prefetch_train.bits.isMisalign := false.B
1432  io.prefetch_train.bits.hasException := false.B
1433  io.s1_prefetch_spec := s1_fire
1434  io.s2_prefetch_spec := s2_prefetch_train_valid
1435
1436  val s2_prefetch_train_l1_valid = WireInit(false.B)
1437  s2_prefetch_train_l1_valid              := s2_valid && !s2_actually_uncache
1438  io.prefetch_train_l1.valid              := GatedValidRegNext(s2_prefetch_train_l1_valid)
1439  io.prefetch_train_l1.bits.fromLsPipelineBundle(s2_in, latch = true, enable = s2_prefetch_train_l1_valid)
1440  io.prefetch_train_l1.bits.miss          := RegEnable(io.dcache.resp.bits.miss, s2_prefetch_train_l1_valid)
1441  io.prefetch_train_l1.bits.meta_prefetch := RegEnable(io.dcache.resp.bits.meta_prefetch, s2_prefetch_train_l1_valid)
1442  io.prefetch_train_l1.bits.meta_access   := RegEnable(io.dcache.resp.bits.meta_access, s2_prefetch_train_l1_valid)
1443  io.prefetch_train_l1.bits.isFinalSplit      := false.B
1444  io.prefetch_train_l1.bits.misalignWith16Byte := false.B
1445  io.prefetch_train_l1.bits.misalignNeedWakeUp := false.B
1446  io.prefetch_train_l1.bits.updateAddrValid := false.B
1447  io.prefetch_train_l1.bits.hasException := false.B
1448  io.prefetch_train_l1.bits.isMisalign := false.B
1449  if (env.FPGAPlatform){
1450    io.dcache.s0_pc := DontCare
1451    io.dcache.s1_pc := DontCare
1452    io.dcache.s2_pc := DontCare
1453  }else{
1454    io.dcache.s0_pc := s0_out.uop.pc
1455    io.dcache.s1_pc := s1_out.uop.pc
1456    io.dcache.s2_pc := s2_out.uop.pc
1457  }
1458  io.dcache.s2_kill := s2_pmp.ld || s2_actually_uncache || s2_kill
1459
1460  val s1_ld_left_fire = s1_valid && !s1_kill && s2_ready
1461  val s2_ld_valid_dup = RegInit(0.U(6.W))
1462  s2_ld_valid_dup := 0x0.U(6.W)
1463  when (s1_ld_left_fire && !s1_out.isHWPrefetch) { s2_ld_valid_dup := 0x3f.U(6.W) }
1464  when (s1_kill || s1_out.isHWPrefetch) { s2_ld_valid_dup := 0x0.U(6.W) }
1465  assert(RegNext((s2_valid === s2_ld_valid_dup(0)) || RegNext(s1_out.isHWPrefetch)))
1466
1467  // Pipeline
1468  // --------------------------------------------------------------------------------
1469  // stage 3
1470  // --------------------------------------------------------------------------------
1471  // writeback and update load queue
1472  val s3_valid        = GatedValidRegNext(s2_valid && !s2_out.isHWPrefetch && !s2_out.uop.robIdx.needFlush(io.redirect))
1473  val s3_in           = RegEnable(s2_out, s2_fire)
1474  val s3_out          = Wire(Valid(new MemExuOutput))
1475  val s3_dcache_rep   = RegEnable(s2_dcache_fast_rep && s2_troublem, false.B, s2_fire)
1476  val s3_ld_valid_dup = RegEnable(s2_ld_valid_dup, s2_fire)
1477  val s3_fast_rep     = Wire(Bool())
1478  val s3_nc_with_data = RegNext(s2_nc_with_data)
1479  val s3_troublem     = GatedValidRegNext(s2_troublem)
1480  val s3_kill         = s3_in.uop.robIdx.needFlush(io.redirect)
1481  val s3_vecout       = Wire(new OnlyVecExuOutput)
1482  val s3_vecActive    = RegEnable(s2_out.vecActive, true.B, s2_fire)
1483  val s3_isvec        = RegEnable(s2_out.isvec, false.B, s2_fire)
1484  val s3_vec_alignedType = RegEnable(s2_out.alignedType, s2_fire)
1485  val s3_vec_mBIndex     = RegEnable(s2_out.mbIndex, s2_fire)
1486  val s3_frm_mabuf       = s3_in.isFrmMisAlignBuf
1487  val s3_mmio_req     = RegNext(s2_mmio_req)
1488  val s3_pdest        = RegNext(Mux(s2_valid, s2_out.uop.pdest, s2_mmio_req.bits.uop.pdest))
1489  val s3_rfWen        = RegEnable(Mux(s2_valid, s2_out.uop.rfWen, s2_mmio_req.bits.uop.rfWen), s2_valid || s2_mmio_req.valid)
1490  val s3_fpWen        = RegEnable(Mux(s2_valid, s2_out.uop.fpWen, s2_mmio_req.bits.uop.fpWen), s2_valid || s2_mmio_req.valid)
1491  val s3_data_select  = RegEnable(s2_data_select, 0.U(s2_data_select.getWidth.W), s2_fire)
1492  val s3_data_select_by_offset = RegEnable(s2_data_select_by_offset, 0.U.asTypeOf(s2_data_select_by_offset), s2_fire)
1493  val s3_hw_err   =
1494      if (EnableAccurateLoadError) {
1495        io.dcache.resp.bits.error_delayed && GatedValidRegNext(io.csrCtrl.cache_error_enable) && s3_troublem
1496      } else {
1497        WireInit(false.B)
1498      }
1499  val s3_safe_wakeup  = RegEnable(s2_safe_wakeup, s2_fire)
1500  val s3_safe_writeback = RegEnable(s2_safe_writeback, s2_fire) || s3_hw_err
1501  val s3_exception = RegEnable(s2_real_exception, s2_fire)
1502  val s3_mis_align = RegEnable(s2_mis_align, s2_fire)
1503  val s3_trigger_debug_mode = RegEnable(s2_trigger_debug_mode, false.B, s2_fire)
1504
1505  // TODO: Fix vector load merge buffer nack
1506  val s3_vec_mb_nack  = Wire(Bool())
1507  s3_vec_mb_nack     := false.B
1508  XSError(s3_valid && s3_vec_mb_nack, "Merge buffer should always accept vector loads!")
1509
1510  s3_ready := !s3_valid || s3_kill || io.ldout.ready
1511
1512
1513  // forwrad last beat
1514  val s3_fast_rep_canceled = io.replay.valid && io.replay.bits.forward_tlDchannel || io.misalign_ldin.valid || !io.dcache.req.ready
1515
1516  val s3_can_enter_lsq_valid = s3_valid && (!s3_fast_rep || s3_fast_rep_canceled) && !s3_in.feedbacked && !s3_nc_with_data && !s3_in.misalignNeedWakeUp
1517  io.lsq.ldin.valid := s3_can_enter_lsq_valid
1518  // TODO: check this --by hx
1519  // io.lsq.ldin.valid := s3_valid && (!s3_fast_rep || !io.fast_rep_out.ready) && !s3_in.feedbacked && !s3_in.lateKill
1520  io.lsq.ldin.bits := s3_in
1521  io.lsq.ldin.bits.miss := s3_in.miss
1522
1523  // connect to misalignBuffer
1524  val toMisalignBufferValid = s3_can_enter_lsq_valid && s3_mis_align && !s3_frm_mabuf
1525  io.misalign_buf.valid := toMisalignBufferValid
1526  io.misalign_buf.bits  := s3_in
1527
1528  /* <------- DANGEROUS: Don't change sequence here ! -------> */
1529  io.lsq.ldin.bits.data_wen_dup := s3_ld_valid_dup.asBools
1530  io.lsq.ldin.bits.replacementUpdated := io.dcache.resp.bits.replacementUpdated
1531  io.lsq.ldin.bits.missDbUpdated := GatedValidRegNext(s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated)
1532  io.lsq.ldin.bits.updateAddrValid := !s3_mis_align && (!s3_frm_mabuf || s3_in.isFinalSplit) || s3_exception
1533  io.lsq.ldin.bits.hasException := false.B
1534
1535  io.s3_dly_ld_err := false.B // s3_dly_ld_err && s3_valid
1536  io.lsq.ldin.bits.dcacheRequireReplay  := s3_dcache_rep
1537
1538  val s3_vp_match_fail = GatedValidRegNext(s2_fwd_vp_match_invalid) && s3_troublem
1539  val s3_rep_frm_fetch = s3_vp_match_fail
1540  val s3_ldld_rep_inst =
1541      io.lsq.ldld_nuke_query.resp.valid &&
1542      io.lsq.ldld_nuke_query.resp.bits.rep_frm_fetch &&
1543      GatedValidRegNext(io.csrCtrl.ldld_vio_check_enable)
1544  val s3_flushPipe = s3_ldld_rep_inst
1545
1546  val s3_lrq_rep_info = WireInit(s3_in.rep_info)
1547  s3_lrq_rep_info.misalign_nack := toMisalignBufferValid && !io.misalign_buf.ready
1548  val s3_lrq_sel_rep_cause = PriorityEncoderOH(s3_lrq_rep_info.cause.asUInt)
1549  val s3_replayqueue_rep_cause = WireInit(0.U.asTypeOf(s3_in.rep_info.cause))
1550  s3_replayqueue_rep_cause(LoadReplayCauses.C_MF) := s3_mis_align && s3_lrq_rep_info.misalign_nack
1551
1552  val s3_mab_rep_info = WireInit(s3_in.rep_info)
1553  val s3_mab_sel_rep_cause = PriorityEncoderOH(s3_mab_rep_info.cause.asUInt)
1554  val s3_misalign_rep_cause = WireInit(0.U.asTypeOf(s3_in.rep_info.cause))
1555
1556  s3_misalign_rep_cause := Mux(
1557    s3_in.misalignNeedWakeUp,
1558    0.U.asTypeOf(s3_mab_rep_info.cause.cloneType),
1559    VecInit(s3_mab_sel_rep_cause.asBools)
1560  )
1561
1562  when (s3_exception || s3_hw_err || s3_rep_frm_fetch || s3_frm_mabuf) {
1563    s3_replayqueue_rep_cause := 0.U.asTypeOf(s3_lrq_rep_info.cause.cloneType)
1564  } .otherwise {
1565    s3_replayqueue_rep_cause := VecInit(s3_lrq_sel_rep_cause.asBools)
1566
1567  }
1568  io.lsq.ldin.bits.rep_info.cause := s3_replayqueue_rep_cause
1569
1570
1571  // Int load, if hit, will be writebacked at s3
1572  s3_out.valid                := s3_valid && s3_safe_writeback && !toMisalignBufferValid
1573  s3_out.bits.uop             := s3_in.uop
1574  s3_out.bits.uop.fpWen       := s3_in.uop.fpWen
1575  s3_out.bits.uop.exceptionVec(loadAccessFault) := s3_in.uop.exceptionVec(loadAccessFault) && s3_vecActive
1576  s3_out.bits.uop.exceptionVec(hardwareError) := s3_hw_err && s3_vecActive
1577  s3_out.bits.uop.flushPipe   := false.B
1578  s3_out.bits.uop.replayInst  := false.B
1579  s3_out.bits.data            := s3_in.data
1580  s3_out.bits.isFromLoadUnit  := true.B
1581  s3_out.bits.debug.isMMIO    := s3_in.mmio
1582  s3_out.bits.debug.isNC      := s3_in.nc
1583  s3_out.bits.debug.isPerfCnt := false.B
1584  s3_out.bits.debug.paddr     := s3_in.paddr
1585  s3_out.bits.debug.vaddr     := s3_in.vaddr
1586
1587  // Vector load, writeback to merge buffer
1588  // TODO: Add assertion in merge buffer, merge buffer must accept vec load writeback
1589  s3_vecout.isvec             := s3_isvec
1590  s3_vecout.vecdata           := 0.U // Data will be assigned later
1591  s3_vecout.mask              := s3_in.mask
1592  // s3_vecout.rob_idx_valid     := s3_in.rob_idx_valid
1593  // s3_vecout.inner_idx         := s3_in.inner_idx
1594  // s3_vecout.rob_idx           := s3_in.rob_idx
1595  // s3_vecout.offset            := s3_in.offset
1596  s3_vecout.reg_offset        := s3_in.reg_offset
1597  s3_vecout.vecActive         := s3_vecActive
1598  s3_vecout.is_first_ele      := s3_in.is_first_ele
1599  // s3_vecout.uopQueuePtr       := DontCare // uopQueuePtr is already saved in flow queue
1600  // s3_vecout.flowPtr           := s3_in.flowPtr
1601  s3_vecout.elemIdx           := s3_in.elemIdx // elemIdx is already saved in flow queue // TODO:
1602  s3_vecout.elemIdxInsideVd   := s3_in.elemIdxInsideVd
1603  s3_vecout.trigger           := s3_in.uop.trigger
1604  s3_vecout.vstart            := s3_in.uop.vpu.vstart
1605  s3_vecout.vecTriggerMask    := s3_in.vecTriggerMask
1606  val s3_usSecondInv          = s3_in.usSecondInv
1607
1608  val s3_frm_mis_flush     = s3_frm_mabuf &&
1609    (io.misalign_ldout.bits.rep_info.fwd_fail || io.misalign_ldout.bits.rep_info.mem_amb || io.misalign_ldout.bits.rep_info.nuke)
1610
1611  io.rollback.valid := s3_valid && (s3_rep_frm_fetch || s3_flushPipe || s3_frm_mis_flush) && !s3_exception
1612  io.rollback.bits             := DontCare
1613  io.rollback.bits.isRVC       := s3_out.bits.uop.preDecodeInfo.isRVC
1614  io.rollback.bits.robIdx      := s3_out.bits.uop.robIdx
1615  io.rollback.bits.ftqIdx      := s3_out.bits.uop.ftqPtr
1616  io.rollback.bits.ftqOffset   := s3_out.bits.uop.ftqOffset
1617  io.rollback.bits.level       := Mux(s3_rep_frm_fetch || s3_frm_mis_flush, RedirectLevel.flush, RedirectLevel.flushAfter)
1618  io.rollback.bits.cfiUpdate.target := s3_out.bits.uop.pc
1619  io.rollback.bits.debug_runahead_checkpoint_id := s3_out.bits.uop.debugInfo.runahead_checkpoint_id
1620  /* <------- DANGEROUS: Don't change sequence here ! -------> */
1621
1622  io.lsq.ldin.bits.uop := s3_out.bits.uop
1623//  io.lsq.ldin.bits.uop.exceptionVec(loadAddrMisaligned) := Mux(s3_in.onlyMisalignException, false.B, s3_in.uop.exceptionVec(loadAddrMisaligned))
1624
1625  val s3_revoke = s3_exception || io.lsq.ldin.bits.rep_info.need_rep || s3_mis_align
1626  io.lsq.ldld_nuke_query.revoke := s3_revoke
1627  io.lsq.stld_nuke_query.revoke := s3_revoke
1628
1629  // feedback slow
1630  s3_fast_rep := RegNext(s2_fast_rep)
1631
1632  val s3_fb_no_waiting = !s3_in.isLoadReplay &&
1633                        (!(s3_fast_rep && !s3_fast_rep_canceled)) &&
1634                        !s3_in.feedbacked
1635
1636  // feedback: scalar load will send feedback to RS
1637  //           vector load will send signal to VL Merge Buffer, then send feedback at granularity of uops
1638  io.feedback_slow.valid                 := s3_valid && s3_fb_no_waiting && !s3_isvec && !s3_frm_mabuf
1639  io.feedback_slow.bits.hit              := !s3_lrq_rep_info.need_rep || io.lsq.ldin.ready
1640  io.feedback_slow.bits.flushState       := s3_in.ptwBack
1641  io.feedback_slow.bits.robIdx           := s3_in.uop.robIdx
1642  io.feedback_slow.bits.sqIdx            := s3_in.uop.sqIdx
1643  io.feedback_slow.bits.lqIdx            := s3_in.uop.lqIdx
1644  io.feedback_slow.bits.sourceType       := RSFeedbackType.lrqFull
1645  io.feedback_slow.bits.dataInvalidSqIdx := DontCare
1646
1647  // TODO: vector wakeup?
1648  io.ldCancel.ld2Cancel := s3_valid && !s3_safe_wakeup && !s3_isvec && (!s3_frm_mabuf || s3_in.misalignNeedWakeUp)
1649
1650  val s3_ld_wb_meta = Mux(s3_valid, s3_out.bits, s3_mmio_req.bits)
1651
1652  // data from load queue refill
1653  val s3_ld_raw_data_frm_mmio = RegNextN(io.lsq.ld_raw_data, 3)
1654  val s3_merged_data_frm_mmio = s3_ld_raw_data_frm_mmio.mergedData()
1655  val s3_picked_data_frm_mmio = LookupTree(s3_ld_raw_data_frm_mmio.addrOffset, List(
1656    "b000".U -> s3_merged_data_frm_mmio(63,  0),
1657    "b001".U -> s3_merged_data_frm_mmio(63,  8),
1658    "b010".U -> s3_merged_data_frm_mmio(63, 16),
1659    "b011".U -> s3_merged_data_frm_mmio(63, 24),
1660    "b100".U -> s3_merged_data_frm_mmio(63, 32),
1661    "b101".U -> s3_merged_data_frm_mmio(63, 40),
1662    "b110".U -> s3_merged_data_frm_mmio(63, 48),
1663    "b111".U -> s3_merged_data_frm_mmio(63, 56)
1664  ))
1665  val s3_ld_data_frm_mmio = rdataHelper(s3_ld_raw_data_frm_mmio.uop, s3_picked_data_frm_mmio)
1666
1667  /* data from pipe, which forward from respectively
1668   *  dcache hit: [D channel, mshr, sbuffer, sq]
1669   *  nc_with_data: [sq]
1670   */
1671
1672  val s2_ld_data_frm_nc = shiftDataToHigh(s2_out.paddr, s2_out.data)
1673  val s2_ld_raw_data_frm_pipe = Wire(new LoadDataFromDcacheBundle)
1674  s2_ld_raw_data_frm_pipe.respDcacheData       := Mux(s2_nc_with_data, s2_ld_data_frm_nc, io.dcache.resp.bits.data)
1675  s2_ld_raw_data_frm_pipe.forward_D            := s2_fwd_frm_d_chan && !s2_nc_with_data
1676  s2_ld_raw_data_frm_pipe.forwardData_D        := s2_fwd_data_frm_d_chan
1677  s2_ld_raw_data_frm_pipe.forward_mshr         := s2_fwd_frm_mshr && !s2_nc_with_data
1678  s2_ld_raw_data_frm_pipe.forwardData_mshr     := s2_fwd_data_frm_mshr
1679  s2_ld_raw_data_frm_pipe.forward_result_valid := s2_fwd_data_valid
1680
1681  s2_ld_raw_data_frm_pipe.forwardMask          := s2_fwd_mask
1682  s2_ld_raw_data_frm_pipe.forwardData          := s2_fwd_data
1683  s2_ld_raw_data_frm_pipe.uop                  := s2_out.uop
1684  s2_ld_raw_data_frm_pipe.addrOffset           := s2_out.paddr(3, 0)
1685
1686  val s2_ld_raw_data_frm_tlD = s2_ld_raw_data_frm_pipe.mergeTLData()
1687  val s2_merged_data_frm_pipe = s2_ld_raw_data_frm_pipe.mergeLsqFwdData(s2_ld_raw_data_frm_tlD)
1688  val s3_merged_data_frm_pipe = RegEnable(s2_merged_data_frm_pipe, s2_fire)
1689
1690  // duplicate reg for ldout and vecldout
1691  private val LdDataDup = 3
1692  require(LdDataDup >= 2)
1693
1694  val s3_data_frm_pipe = VecInit((0 until LdDataDup).map(i => {
1695    VecInit(Seq(
1696      s3_merged_data_frm_pipe(63,      0),
1697      s3_merged_data_frm_pipe(71,      8),
1698      s3_merged_data_frm_pipe(79,     16),
1699      s3_merged_data_frm_pipe(87,     24),
1700      s3_merged_data_frm_pipe(95,     32),
1701      s3_merged_data_frm_pipe(103,    40),
1702      s3_merged_data_frm_pipe(111,    48),
1703      s3_merged_data_frm_pipe(119,    56),
1704      s3_merged_data_frm_pipe(127,    64),
1705      s3_merged_data_frm_pipe(127,    72),
1706      s3_merged_data_frm_pipe(127,    80),
1707      s3_merged_data_frm_pipe(127,    88),
1708      s3_merged_data_frm_pipe(127,    96),
1709      s3_merged_data_frm_pipe(127,   104),
1710      s3_merged_data_frm_pipe(127,   112),
1711      s3_merged_data_frm_pipe(127,   120),
1712    ))
1713  }))
1714  val s3_picked_data_frm_pipe = VecInit((0 until LdDataDup).map(i => {
1715    Mux1H(s3_data_select_by_offset, s3_data_frm_pipe(i))
1716  }))
1717  val s3_ld_data_frm_pipe = VecInit((0 until LdDataDup).map(i => {
1718    newRdataHelper(s3_data_select, s3_picked_data_frm_pipe(i))
1719  }))
1720
1721  // FIXME: add 1 cycle delay ?
1722  // io.lsq.uncache.ready := !s3_valid
1723  val s3_ldout_valid  = s3_mmio_req.valid ||
1724                        s3_out.valid && RegNext(!s2_out.isvec && !s2_out.isFrmMisAlignBuf)
1725  val s3_outexception = ExceptionNO.selectByFu(s3_out.bits.uop.exceptionVec, LduCfg).asUInt.orR && s3_vecActive
1726  io.ldout.valid       := s3_ldout_valid
1727  io.ldout.bits        := s3_ld_wb_meta
1728  io.ldout.bits.data   := Mux(s3_valid, s3_ld_data_frm_pipe(0), s3_ld_data_frm_mmio)
1729  io.ldout.bits.uop.rfWen := s3_rfWen
1730  io.ldout.bits.uop.fpWen := s3_fpWen
1731  io.ldout.bits.uop.pdest := s3_pdest
1732  io.ldout.bits.uop.exceptionVec := ExceptionNO.selectByFu(s3_ld_wb_meta.uop.exceptionVec, LduCfg)
1733  io.ldout.bits.isFromLoadUnit := true.B
1734  io.ldout.bits.uop.fuType := Mux(
1735                                  s3_valid && s3_isvec,
1736                                  FuType.vldu.U,
1737                                  FuType.ldu.U
1738  )
1739
1740  XSError(s3_valid && s3_in.misalignNeedWakeUp && !s3_frm_mabuf, "Only the needwakeup from the misalignbuffer may be high")
1741  XSError(s3_valid && s3_vecout.isvec && s3_in.vecActive && !s3_vecout.mask.orR, "In vecActive, mask complement should not be 0")
1742  // TODO: check this --hx
1743  // io.ldout.valid       := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) && !s3_vecout.isvec ||
1744  //   io.lsq.uncache.valid && !io.lsq.uncache.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid && !io.lsq.uncache.bits.isVls
1745  //  io.ldout.bits.data   := Mux(s3_out.valid, s3_ld_data_frm_pipe, s3_ld_data_frm_mmio)
1746  //  io.ldout.valid       := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) ||
1747  //                         s3_mmio_req.valid && !s3_mmio_req.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid
1748
1749  // s3 load fast replay
1750  io.fast_rep_out.valid := s3_valid && s3_fast_rep
1751  io.fast_rep_out.bits := s3_in
1752  io.fast_rep_out.bits.lateKill := s3_rep_frm_fetch
1753  io.fast_rep_out.bits.delayedLoadError := s3_hw_err
1754
1755  val vecFeedback = s3_valid && s3_fb_no_waiting && s3_lrq_rep_info.need_rep && !io.lsq.ldin.ready && s3_isvec
1756
1757  // vector output
1758  io.vecldout.bits.alignedType := s3_vec_alignedType
1759  // vec feedback
1760  io.vecldout.bits.vecFeedback := vecFeedback
1761  // TODO: VLSU, uncache data logic
1762  val vecdata = rdataVecHelper(s3_vec_alignedType(1,0), s3_picked_data_frm_pipe(1))
1763  io.vecldout.bits.vecdata.get := Mux(
1764    s3_in.misalignWith16Byte,
1765    s3_picked_data_frm_pipe(1),
1766    Mux(
1767      s3_in.is128bit,
1768      s3_merged_data_frm_pipe,
1769      vecdata
1770    )
1771  )
1772  io.vecldout.bits.isvec := s3_vecout.isvec
1773  io.vecldout.bits.elemIdx := s3_vecout.elemIdx
1774  io.vecldout.bits.elemIdxInsideVd.get := s3_vecout.elemIdxInsideVd
1775  io.vecldout.bits.mask := s3_vecout.mask
1776  io.vecldout.bits.hasException := s3_exception
1777  io.vecldout.bits.reg_offset.get := s3_vecout.reg_offset
1778  io.vecldout.bits.usSecondInv := s3_usSecondInv
1779  io.vecldout.bits.mBIndex := s3_vec_mBIndex
1780  io.vecldout.bits.hit := !s3_lrq_rep_info.need_rep || io.lsq.ldin.ready
1781  io.vecldout.bits.sourceType := RSFeedbackType.lrqFull
1782  io.vecldout.bits.trigger := s3_vecout.trigger
1783  io.vecldout.bits.flushState := DontCare
1784  io.vecldout.bits.exceptionVec := ExceptionNO.selectByFu(s3_out.bits.uop.exceptionVec, VlduCfg)
1785  io.vecldout.bits.vaddr := s3_in.fullva
1786  io.vecldout.bits.vaNeedExt := s3_in.vaNeedExt
1787  io.vecldout.bits.gpaddr := s3_in.gpaddr
1788  io.vecldout.bits.isForVSnonLeafPTE := s3_in.isForVSnonLeafPTE
1789  io.vecldout.bits.mmio := DontCare
1790  io.vecldout.bits.vstart := s3_vecout.vstart
1791  io.vecldout.bits.vecTriggerMask := s3_vecout.vecTriggerMask
1792  io.vecldout.bits.nc := DontCare
1793
1794  io.vecldout.valid := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) && s3_vecout.isvec && !s3_mis_align && !s3_frm_mabuf //||
1795  // TODO: check this, why !io.lsq.uncache.bits.isVls before?
1796  // Now vector instruction don't support mmio.
1797    // io.lsq.uncache.valid && !io.lsq.uncache.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid && io.lsq.uncache.bits.isVls
1798    //io.lsq.uncache.valid && !io.lsq.uncache.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid && !io.lsq.uncache.bits.isVls
1799
1800  io.misalign_ldout.valid     := s3_valid && (!s3_fast_rep || s3_fast_rep_canceled) && s3_frm_mabuf
1801  io.misalign_ldout.bits      := io.lsq.ldin.bits
1802  io.misalign_ldout.bits.data := Mux(s3_in.misalignWith16Byte, s3_merged_data_frm_pipe, s3_picked_data_frm_pipe(2))
1803  io.misalign_ldout.bits.rep_info.cause := s3_misalign_rep_cause
1804
1805  // fast load to load forward
1806  if (EnableLoadToLoadForward) {
1807    io.l2l_fwd_out.valid      := s3_valid && !s3_in.mmio && !s3_in.nc && !s3_lrq_rep_info.need_rep
1808    io.l2l_fwd_out.data       := Mux(s3_in.vaddr(3), s3_merged_data_frm_pipe(127, 64), s3_merged_data_frm_pipe(63, 0))
1809    io.l2l_fwd_out.dly_ld_err := s3_hw_err || // ecc delayed error
1810                                 s3_ldld_rep_inst ||
1811                                 s3_rep_frm_fetch
1812  } else {
1813    io.l2l_fwd_out.valid := false.B
1814    io.l2l_fwd_out.data := DontCare
1815    io.l2l_fwd_out.dly_ld_err := DontCare
1816  }
1817
1818  // s1
1819  io.debug_ls.s1_robIdx := s1_in.uop.robIdx.value
1820  io.debug_ls.s1_isLoadToLoadForward := s1_fire && s1_try_ptr_chasing && !s1_ptr_chasing_canceled
1821  io.debug_ls.s1_isTlbFirstMiss := s1_fire && s1_tlb_miss && s1_in.isFirstIssue
1822  // s2
1823  io.debug_ls.s2_robIdx := s2_in.uop.robIdx.value
1824  io.debug_ls.s2_isBankConflict := s2_fire && (!s2_kill && s2_bank_conflict)
1825  io.debug_ls.s2_isDcacheFirstMiss := s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue
1826  io.debug_ls.s2_isForwardFail := s2_fire && s2_fwd_fail
1827  // s3
1828  io.debug_ls.s3_robIdx := s3_in.uop.robIdx.value
1829  io.debug_ls.s3_isReplayFast := s3_valid && s3_fast_rep && !s3_fast_rep_canceled
1830  io.debug_ls.s3_isReplayRS :=  RegNext(io.feedback_fast.valid && !io.feedback_fast.bits.hit) || (io.feedback_slow.valid && !io.feedback_slow.bits.hit)
1831  io.debug_ls.s3_isReplaySlow := io.lsq.ldin.valid && io.lsq.ldin.bits.rep_info.need_rep
1832  io.debug_ls.s3_isReplay := s3_valid && s3_lrq_rep_info.need_rep // include fast+slow+rs replay
1833  io.debug_ls.replayCause := s3_lrq_rep_info.cause
1834  io.debug_ls.replayCnt := 1.U
1835
1836  // Topdown
1837  io.lsTopdownInfo.s1.robIdx          := s1_in.uop.robIdx.value
1838  io.lsTopdownInfo.s1.vaddr_valid     := s1_valid && s1_in.hasROBEntry
1839  io.lsTopdownInfo.s1.vaddr_bits      := s1_vaddr
1840  io.lsTopdownInfo.s2.robIdx          := s2_in.uop.robIdx.value
1841  io.lsTopdownInfo.s2.paddr_valid     := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss
1842  io.lsTopdownInfo.s2.paddr_bits      := s2_in.paddr
1843  io.lsTopdownInfo.s2.first_real_miss := io.dcache.resp.bits.real_miss
1844  io.lsTopdownInfo.s2.cache_miss_en   := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated
1845
1846  // perf cnt
1847  XSPerfAccumulate("s0_in_valid",                  io.ldin.valid)
1848  XSPerfAccumulate("s0_in_block",                  io.ldin.valid && !io.ldin.fire)
1849  XSPerfAccumulate("s0_vecin_valid",               io.vecldin.valid)
1850  XSPerfAccumulate("s0_vecin_block",               io.vecldin.valid && !io.vecldin.fire)
1851  XSPerfAccumulate("s0_in_fire_first_issue",       s0_valid && s0_sel_src.isFirstIssue)
1852  XSPerfAccumulate("s0_lsq_replay_issue",          io.replay.fire)
1853  XSPerfAccumulate("s0_lsq_replay_vecissue",       io.replay.fire && io.replay.bits.isvec)
1854  XSPerfAccumulate("s0_ldu_fire_first_issue",      io.ldin.fire && s0_sel_src.isFirstIssue)
1855  XSPerfAccumulate("s0_fast_replay_issue",         io.fast_rep_in.fire)
1856  XSPerfAccumulate("s0_fast_replay_vecissue",      io.fast_rep_in.fire && io.fast_rep_in.bits.isvec)
1857  XSPerfAccumulate("s0_stall_out",                 s0_valid && !s0_can_go)
1858  XSPerfAccumulate("s0_stall_dcache",              s0_valid && !io.dcache.req.ready)
1859  XSPerfAccumulate("s0_addr_spec_success",         s0_fire && s0_dcache_vaddr(VAddrBits-1, 12) === io.ldin.bits.src(0)(VAddrBits-1, 12))
1860  XSPerfAccumulate("s0_addr_spec_failed",          s0_fire && s0_dcache_vaddr(VAddrBits-1, 12) =/= io.ldin.bits.src(0)(VAddrBits-1, 12))
1861  XSPerfAccumulate("s0_addr_spec_success_once",    s0_fire && s0_dcache_vaddr(VAddrBits-1, 12) === io.ldin.bits.src(0)(VAddrBits-1, 12) && s0_sel_src.isFirstIssue)
1862  XSPerfAccumulate("s0_addr_spec_failed_once",     s0_fire && s0_dcache_vaddr(VAddrBits-1, 12) =/= io.ldin.bits.src(0)(VAddrBits-1, 12) && s0_sel_src.isFirstIssue)
1863  XSPerfAccumulate("s0_vec_addr_vlen_aligned",     s0_fire && s0_sel_src.isvec && s0_dcache_vaddr(3, 0) === 0.U)
1864  XSPerfAccumulate("s0_vec_addr_vlen_unaligned",   s0_fire && s0_sel_src.isvec && s0_dcache_vaddr(3, 0) =/= 0.U)
1865  XSPerfAccumulate("s0_forward_tl_d_channel",      s0_out.forward_tlDchannel)
1866  XSPerfAccumulate("s0_hardware_prefetch_fire",    s0_fire && s0_hw_prf_select)
1867  XSPerfAccumulate("s0_software_prefetch_fire",    s0_fire && s0_sel_src.prf && s0_src_select_vec(int_iss_idx))
1868  XSPerfAccumulate("s0_hardware_prefetch_blocked", io.prefetch_req.valid && !s0_hw_prf_select)
1869  XSPerfAccumulate("s0_hardware_prefetch_total",   io.prefetch_req.valid)
1870
1871  XSPerfAccumulate("s3_rollback_total",             io.rollback.valid)
1872  XSPerfAccumulate("s3_rep_frm_fetch_rollback",     io.rollback.valid && s3_rep_frm_fetch)
1873  XSPerfAccumulate("s3_flushPipe_rollback",         io.rollback.valid && s3_flushPipe)
1874  XSPerfAccumulate("s3_frm_mis_flush_rollback",     io.rollback.valid && s3_frm_mis_flush)
1875
1876  XSPerfAccumulate("s1_in_valid",                  s1_valid)
1877  XSPerfAccumulate("s1_in_fire",                   s1_fire)
1878  XSPerfAccumulate("s1_in_fire_first_issue",       s1_fire && s1_in.isFirstIssue)
1879  XSPerfAccumulate("s1_tlb_miss",                  s1_fire && s1_tlb_miss)
1880  XSPerfAccumulate("s1_tlb_miss_first_issue",      s1_fire && s1_tlb_miss && s1_in.isFirstIssue)
1881  XSPerfAccumulate("s1_stall_out",                 s1_valid && !s1_can_go)
1882  XSPerfAccumulate("s1_dly_err",                   s1_valid && s1_fast_rep_dly_err)
1883
1884  XSPerfAccumulate("s2_in_valid",                  s2_valid)
1885  XSPerfAccumulate("s2_in_fire",                   s2_fire)
1886  XSPerfAccumulate("s2_in_fire_first_issue",       s2_fire && s2_in.isFirstIssue)
1887  XSPerfAccumulate("s2_dcache_miss",               s2_fire && io.dcache.resp.bits.miss)
1888  XSPerfAccumulate("s2_dcache_miss_first_issue",   s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue)
1889  XSPerfAccumulate("s2_dcache_real_miss_first_issue",   s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue)
1890  XSPerfAccumulate("s2_full_forward",              s2_fire && s2_full_fwd)
1891  XSPerfAccumulate("s2_dcache_miss_full_forward",  s2_fire && s2_dcache_miss)
1892  XSPerfAccumulate("s2_fwd_frm_d_can",             s2_valid && s2_fwd_frm_d_chan)
1893  XSPerfAccumulate("s2_fwd_frm_d_chan_or_mshr",    s2_valid && s2_fwd_frm_d_chan_or_mshr)
1894  XSPerfAccumulate("s2_stall_out",                 s2_fire && !s2_can_go)
1895  XSPerfAccumulate("s2_prefetch",                  s2_fire && s2_prf)
1896  XSPerfAccumulate("s2_prefetch_ignored",          s2_fire && s2_prf && io.dcache.s2_mq_nack) // ignore prefetch for mshr full / miss req port conflict
1897  XSPerfAccumulate("s2_prefetch_miss",             s2_fire && s2_prf && io.dcache.resp.bits.miss) // prefetch req miss in l1
1898  XSPerfAccumulate("s2_prefetch_hit",              s2_fire && s2_prf && !io.dcache.resp.bits.miss) // prefetch req hit in l1
1899  XSPerfAccumulate("s2_prefetch_accept",           s2_fire && s2_prf && io.dcache.resp.bits.miss && !io.dcache.s2_mq_nack) // prefetch a missed line in l1, and l1 accepted it
1900  XSPerfAccumulate("s2_forward_req",               s2_fire && s2_in.forward_tlDchannel)
1901  XSPerfAccumulate("s2_successfully_forward_channel_D", s2_fire && s2_fwd_frm_d_chan && s2_fwd_data_valid)
1902  XSPerfAccumulate("s2_successfully_forward_mshr",      s2_fire && s2_fwd_frm_mshr && s2_fwd_data_valid)
1903
1904  XSPerfAccumulate("load_to_load_forward",                      s1_try_ptr_chasing && !s1_ptr_chasing_canceled)
1905  XSPerfAccumulate("load_to_load_forward_try",                  s1_try_ptr_chasing)
1906  XSPerfAccumulate("load_to_load_forward_fail",                 s1_cancel_ptr_chasing)
1907  XSPerfAccumulate("load_to_load_forward_fail_cancelled",       s1_cancel_ptr_chasing && s1_ptr_chasing_canceled)
1908  XSPerfAccumulate("load_to_load_forward_fail_wakeup_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && s1_not_fast_match)
1909  XSPerfAccumulate("load_to_load_forward_fail_op_not_ld",       s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && s1_fu_op_type_not_ld)
1910  XSPerfAccumulate("load_to_load_forward_fail_addr_align",      s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && s1_addr_misaligned)
1911  XSPerfAccumulate("load_to_load_forward_fail_set_mismatch",    s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && !s1_addr_misaligned && s1_addr_mismatch)
1912
1913  XSPerfAccumulate("nc_ld_writeback", io.ldout.valid && s3_nc_with_data)
1914  XSPerfAccumulate("nc_ld_exception", s3_valid && s3_nc_with_data && s3_in.uop.exceptionVec.reduce(_ || _))
1915  XSPerfAccumulate("nc_ldld_vio", s3_valid && s3_nc_with_data && s3_ldld_rep_inst)
1916  XSPerfAccumulate("nc_stld_vio", s3_valid && s3_nc_with_data && s3_in.rep_info.nuke)
1917  XSPerfAccumulate("nc_ldld_vioNack", s3_valid && s3_nc_with_data && s3_in.rep_info.rar_nack)
1918  XSPerfAccumulate("nc_stld_vioNack", s3_valid && s3_nc_with_data && s3_in.rep_info.raw_nack)
1919  XSPerfAccumulate("nc_stld_fwd", s3_valid && s3_nc_with_data && RegNext(s2_full_fwd))
1920  XSPerfAccumulate("nc_stld_fwdNotReady", s3_valid && s3_nc_with_data && RegNext(s2_mem_amb || s2_fwd_fail))
1921  XSPerfAccumulate("nc_stld_fwdAddrMismatch", s3_valid && s3_nc_with_data && s3_vp_match_fail)
1922
1923  // bug lyq: some signals in perfEvents are no longer suitable for the current MemBlock design
1924  // hardware performance counter
1925  val perfEvents = Seq(
1926    ("load_s0_in_fire         ", s0_fire                                                        ),
1927    ("load_to_load_forward    ", s1_fire && s1_try_ptr_chasing && !s1_ptr_chasing_canceled      ),
1928    ("stall_dcache            ", s0_valid && s0_can_go && !io.dcache.req.ready                  ),
1929    ("load_s1_in_fire         ", s0_fire                                                        ),
1930    ("load_s1_tlb_miss        ", s1_fire && io.tlb.resp.bits.miss                               ),
1931    ("load_s2_in_fire         ", s1_fire                                                        ),
1932    ("load_s2_dcache_miss     ", s2_fire && io.dcache.resp.bits.miss                            ),
1933  )
1934  generatePerfEvent()
1935
1936  if (backendParams.debugEn){
1937    dontTouch(s0_src_valid_vec)
1938    dontTouch(s0_src_ready_vec)
1939    dontTouch(s0_src_select_vec)
1940    dontTouch(s3_ld_data_frm_pipe)
1941    s3_data_select_by_offset.map(x=> dontTouch(x))
1942    s3_data_frm_pipe.map(x=> dontTouch(x))
1943    s3_picked_data_frm_pipe.map(x=> dontTouch(x))
1944  }
1945
1946  XSDebug(io.ldout.fire, "ldout %x\n", io.ldout.bits.uop.pc)
1947  // end
1948}
1949