xref: /XiangShan/src/main/scala/xiangshan/cache/dcache/mainpipe/MainPipe.scala (revision cb9c18dceb97309b5910e59ee9b3ae0c2b605866) 
1/***************************************************************************************
2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
3* Copyright (c) 2020-2021 Peng Cheng Laboratory
4*
5* XiangShan is licensed under Mulan PSL v2.
6* You can use this software according to the terms and conditions of the Mulan PSL v2.
7* You may obtain a copy of Mulan PSL v2 at:
8*          http://license.coscl.org.cn/MulanPSL2
9*
10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
13*
14* See the Mulan PSL v2 for more details.
15***************************************************************************************/
16
17package xiangshan.cache
18
19import chipsalliance.rocketchip.config.Parameters
20import chisel3._
21import chisel3.util._
22import freechips.rocketchip.tilelink.ClientStates._
23import freechips.rocketchip.tilelink.MemoryOpCategories._
24import freechips.rocketchip.tilelink.TLPermissions._
25import freechips.rocketchip.tilelink.{ClientMetadata, ClientStates, TLPermissions}
26import utils._
27import xiangshan.L1CacheErrorInfo
28
29class MainPipeReq(implicit p: Parameters) extends DCacheBundle {
30  val miss = Bool() // only amo miss will refill in main pipe
31  val miss_id = UInt(log2Up(cfg.nMissEntries).W)
32  val miss_param = UInt(TLPermissions.bdWidth.W)
33  val miss_dirty = Bool()
34  val miss_way_en = UInt(DCacheWays.W)
35
36  val probe = Bool()
37  val probe_param = UInt(TLPermissions.bdWidth.W)
38  val probe_need_data = Bool()
39
40  // request info
41  // reqs from Store, AMO use this
42  // probe does not use this
43  val source = UInt(sourceTypeWidth.W)
44  val cmd = UInt(M_SZ.W)
45  // if dcache size > 32KB, vaddr is also needed for store
46  // vaddr is used to get extra index bits
47  val vaddr  = UInt(VAddrBits.W)
48  // must be aligned to block
49  val addr   = UInt(PAddrBits.W)
50
51  // store
52  val store_data = UInt((cfg.blockBytes * 8).W)
53  val store_mask = UInt(cfg.blockBytes.W)
54
55  // which word does amo work on?
56  val word_idx = UInt(log2Up(cfg.blockBytes * 8 / DataBits).W)
57  val amo_data   = UInt(DataBits.W)
58  val amo_mask   = UInt((DataBits / 8).W)
59
60  // error
61  val error = Bool()
62
63  // replace
64  val replace = Bool()
65  val replace_way_en = UInt(DCacheWays.W)
66
67  val id = UInt(reqIdWidth.W)
68
69  def isLoad: Bool = source === LOAD_SOURCE.U
70  def isStore: Bool = source === STORE_SOURCE.U
71  def isAMO: Bool = source === AMO_SOURCE.U
72
73  def convertStoreReq(store: DCacheLineReq): MainPipeReq = {
74    val req = Wire(new MainPipeReq)
75    req := DontCare
76    req.miss := false.B
77    req.miss_dirty := false.B
78    req.probe := false.B
79    req.probe_need_data := false.B
80    req.source := STORE_SOURCE.U
81    req.cmd := store.cmd
82    req.addr := store.addr
83    req.vaddr := store.vaddr
84    req.store_data := store.data
85    req.store_mask := store.mask
86    req.replace := false.B
87    req.error := false.B
88    req.id := store.id
89    req
90  }
91}
92
93class MainPipeStatus(implicit p: Parameters) extends DCacheBundle {
94  val set = UInt(idxBits.W)
95  val way_en = UInt(nWays.W)
96}
97
98class MainPipe(implicit p: Parameters) extends DCacheModule with HasPerfEvents {
99  val io = IO(new Bundle() {
100    // probe queue
101    val probe_req = Flipped(DecoupledIO(new MainPipeReq))
102    // store miss go to miss queue
103    val miss_req = DecoupledIO(new MissReq)
104    // store buffer
105    val store_req = Flipped(DecoupledIO(new DCacheLineReq))
106    val store_replay_resp = ValidIO(new DCacheLineResp)
107    val store_hit_resp = ValidIO(new DCacheLineResp)
108    val release_update = ValidIO(new ReleaseUpdate)
109    // atmoics
110    val atomic_req = Flipped(DecoupledIO(new MainPipeReq))
111    val atomic_resp = ValidIO(new AtomicsResp)
112    // replace
113    val replace_req = Flipped(DecoupledIO(new MainPipeReq))
114    val replace_resp = ValidIO(UInt(log2Up(cfg.nMissEntries).W))
115    // write-back queue
116    val wb = DecoupledIO(new WritebackReq)
117    val wb_ready_dup = Vec(nDupWbReady, Input(Bool()))
118
119    val data_read_intend = Output(Bool())
120    val data_read = DecoupledIO(new L1BankedDataReadLineReq)
121    val data_resp = Input(Vec(DCacheBanks, new L1BankedDataReadResult()))
122    val readline_error_delayed = Input(Bool())
123    val data_write = DecoupledIO(new L1BankedDataWriteReq)
124    val data_write_dup = Vec(DCacheBanks, Valid(new L1BankedDataWriteReqCtrl))
125    val data_write_ready_dup = Vec(nDupDataWriteReady, Input(Bool()))
126
127    val meta_read = DecoupledIO(new MetaReadReq)
128    val meta_resp = Input(Vec(nWays, new Meta))
129    val meta_write = DecoupledIO(new MetaWriteReq)
130    val error_flag_resp = Input(Vec(nWays, Bool()))
131    val error_flag_write = DecoupledIO(new ErrorWriteReq)
132
133    val tag_read = DecoupledIO(new TagReadReq)
134    val tag_resp = Input(Vec(nWays, UInt(encTagBits.W)))
135    val tag_write = DecoupledIO(new TagWriteReq)
136    val tag_write_ready_dup = Vec(nDupTagWriteReady, Input(Bool()))
137    val tag_write_intend = Output(new Bool())
138
139    // update state vec in replacement algo
140    val replace_access = ValidIO(new ReplacementAccessBundle)
141    // find the way to be replaced
142    val replace_way = new ReplacementWayReqIO
143
144    val status = new Bundle() {
145      val s0_set = ValidIO(UInt(idxBits.W))
146      val s1, s2, s3 = ValidIO(new MainPipeStatus)
147    }
148    val status_dup = Vec(nDupStatus, new Bundle() {
149      val s1, s2, s3 = ValidIO(new MainPipeStatus)
150    })
151
152    // lrsc locked block should block probe
153    val lrsc_locked_block = Output(Valid(UInt(PAddrBits.W)))
154    val invalid_resv_set = Input(Bool())
155    val update_resv_set = Output(Bool())
156    val block_lr = Output(Bool())
157
158    // ecc error
159    val error = Output(new L1CacheErrorInfo())
160  })
161
162  // meta array is made of regs, so meta write or read should always be ready
163  assert(RegNext(io.meta_read.ready))
164  assert(RegNext(io.meta_write.ready))
165
166  val s1_s0_set_conflict, s2_s0_set_conlict, s3_s0_set_conflict = Wire(Bool())
167  val set_conflict = s1_s0_set_conflict || s2_s0_set_conlict || s3_s0_set_conflict
168  // check sbuffer store req set_conflict in parallel with req arbiter
169  // it will speed up the generation of store_req.ready, which is in crit. path
170  val s1_s0_set_conflict_store, s2_s0_set_conlict_store, s3_s0_set_conflict_store = Wire(Bool())
171  val store_set_conflict = s1_s0_set_conflict_store || s2_s0_set_conlict_store || s3_s0_set_conflict_store
172  val s1_ready, s2_ready, s3_ready = Wire(Bool())
173
174  // convert store req to main pipe req, and select a req from store and probe
175  val store_req = Wire(DecoupledIO(new MainPipeReq))
176  store_req.bits := (new MainPipeReq).convertStoreReq(io.store_req.bits)
177  store_req.valid := io.store_req.valid
178  io.store_req.ready := store_req.ready
179
180  // s0: read meta and tag
181  val req = Wire(DecoupledIO(new MainPipeReq))
182  arbiter(
183    in = Seq(
184      io.probe_req,
185      io.replace_req,
186      store_req, // Note: store_req.ready is now manually assigned for better timing
187      io.atomic_req
188    ),
189    out = req,
190    name = Some("main_pipe_req")
191  )
192
193  val store_idx = get_idx(io.store_req.bits.vaddr)
194  // manually assign store_req.ready for better timing
195  // now store_req set conflict check is done in parallel with req arbiter
196  store_req.ready := io.meta_read.ready && io.tag_read.ready && s1_ready && !store_set_conflict &&
197    !io.probe_req.valid && !io.replace_req.valid
198  val s0_req = req.bits
199  val s0_idx = get_idx(s0_req.vaddr)
200  val s0_need_tag = io.tag_read.valid
201  val s0_can_go = io.meta_read.ready && io.tag_read.ready && s1_ready && !set_conflict
202  val s0_fire = req.valid && s0_can_go
203
204  val bank_write = VecInit((0 until DCacheBanks).map(i => get_mask_of_bank(i, s0_req.store_mask).orR)).asUInt
205  val bank_full_write = VecInit((0 until DCacheBanks).map(i => get_mask_of_bank(i, s0_req.store_mask).andR)).asUInt
206  val banks_full_overwrite = bank_full_write.andR
207
208  val banked_store_rmask = bank_write & ~bank_full_write
209  val banked_full_rmask = ~0.U(DCacheBanks.W)
210  val banked_none_rmask = 0.U(DCacheBanks.W)
211
212  val store_need_data = !s0_req.probe && s0_req.isStore && banked_store_rmask.orR
213  val probe_need_data = s0_req.probe
214  val amo_need_data = !s0_req.probe && s0_req.isAMO
215  val miss_need_data = s0_req.miss
216  val replace_need_data = s0_req.replace
217
218  val banked_need_data = store_need_data || probe_need_data || amo_need_data || miss_need_data || replace_need_data
219
220  val s0_banked_rmask = Mux(store_need_data, banked_store_rmask,
221    Mux(probe_need_data || amo_need_data || miss_need_data || replace_need_data,
222      banked_full_rmask,
223      banked_none_rmask
224    ))
225
226  // generate wmask here and use it in stage 2
227  val banked_store_wmask = bank_write
228  val banked_full_wmask = ~0.U(DCacheBanks.W)
229  val banked_none_wmask = 0.U(DCacheBanks.W)
230
231  // s1: read data
232  val s1_valid = RegInit(false.B)
233  val s1_need_data = RegEnable(banked_need_data, s0_fire)
234  val s1_req = RegEnable(s0_req, s0_fire)
235  val s1_banked_rmask = RegEnable(s0_banked_rmask, s0_fire)
236  val s1_banked_store_wmask = RegEnable(banked_store_wmask, s0_fire)
237  val s1_need_tag = RegEnable(s0_need_tag, s0_fire)
238  val s1_can_go = s2_ready && (io.data_read.ready || !s1_need_data)
239  val s1_fire = s1_valid && s1_can_go
240  val s1_idx = get_idx(s1_req.vaddr)
241
242  // duplicate regs to reduce fanout
243  val s1_valid_dup = RegInit(VecInit(Seq.fill(6)(false.B)))
244  val s1_req_vaddr_dup_for_data_read = RegEnable(s0_req.vaddr, s0_fire)
245  val s1_idx_dup_for_replace_way = RegEnable(get_idx(s0_req.vaddr), s0_fire)
246
247  val s1_valid_dup_for_status = RegInit(VecInit(Seq.fill(nDupStatus)(false.B)))
248
249  when (s0_fire) {
250    s1_valid := true.B
251    s1_valid_dup.foreach(_ := true.B)
252    s1_valid_dup_for_status.foreach(_ := true.B)
253  }.elsewhen (s1_fire) {
254    s1_valid := false.B
255    s1_valid_dup.foreach(_ := false.B)
256    s1_valid_dup_for_status.foreach(_ := false.B)
257  }
258  s1_ready := !s1_valid_dup(0) || s1_can_go
259  s1_s0_set_conflict := s1_valid_dup(1) && s0_idx === s1_idx
260  s1_s0_set_conflict_store := s1_valid_dup(2) && store_idx === s1_idx
261
262  val meta_resp = Wire(Vec(nWays, (new Meta).asUInt()))
263  val tag_resp = Wire(Vec(nWays, UInt(tagBits.W)))
264  val ecc_resp = Wire(Vec(nWays, UInt(eccTagBits.W)))
265  meta_resp := Mux(RegNext(s0_fire), VecInit(io.meta_resp.map(_.asUInt)), RegNext(meta_resp))
266  tag_resp := Mux(RegNext(s0_fire), VecInit(io.tag_resp.map(r => r(tagBits - 1, 0))), RegNext(tag_resp))
267  ecc_resp := Mux(RegNext(s0_fire), VecInit(io.tag_resp.map(r => r(encTagBits - 1, tagBits))), RegNext(ecc_resp))
268  val enc_tag_resp = Wire(io.tag_resp.cloneType)
269  enc_tag_resp := Mux(RegNext(s0_fire), io.tag_resp, RegNext(enc_tag_resp))
270
271  def wayMap[T <: Data](f: Int => T) = VecInit((0 until nWays).map(f))
272  val s1_tag_eq_way = wayMap((w: Int) => tag_resp(w) === get_tag(s1_req.addr)).asUInt
273  val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && Meta(meta_resp(w)).coh.isValid()).asUInt
274  val s1_tag_match = s1_tag_match_way.orR
275
276  val s1_hit_tag = Mux(s1_tag_match, Mux1H(s1_tag_match_way, wayMap(w => tag_resp(w))), get_tag(s1_req.addr))
277  val s1_hit_coh = ClientMetadata(Mux(s1_tag_match, Mux1H(s1_tag_match_way, wayMap(w => meta_resp(w))), 0.U))
278  val s1_encTag = Mux1H(s1_tag_match_way, wayMap((w: Int) => enc_tag_resp(w)))
279  val s1_flag_error = Mux(s1_tag_match, Mux1H(s1_tag_match_way, wayMap(w => io.error_flag_resp(w))), false.B)
280  val s1_l2_error = s1_req.error
281
282  // replacement policy
283  val s1_repl_way_en = WireInit(0.U(nWays.W))
284  s1_repl_way_en := Mux(RegNext(s0_fire), UIntToOH(io.replace_way.way), RegNext(s1_repl_way_en))
285  val s1_repl_tag = Mux1H(s1_repl_way_en, wayMap(w => tag_resp(w)))
286  val s1_repl_coh = Mux1H(s1_repl_way_en, wayMap(w => meta_resp(w))).asTypeOf(new ClientMetadata)
287  val s1_miss_tag = Mux1H(s1_req.miss_way_en, wayMap(w => tag_resp(w)))
288  val s1_miss_coh = Mux1H(s1_req.miss_way_en, wayMap(w => meta_resp(w))).asTypeOf(new ClientMetadata)
289
290  val s1_repl_way_raw = WireInit(0.U(log2Up(nWays).W))
291  s1_repl_way_raw := Mux(RegNext(s0_fire), io.replace_way.way, RegNext(s1_repl_way_raw))
292
293  val s1_need_replacement = (s1_req.miss || s1_req.isStore && !s1_req.probe) && !s1_tag_match
294  val s1_way_en = Mux(
295    s1_req.replace,
296    s1_req.replace_way_en,
297    Mux(
298      s1_req.miss,
299      s1_req.miss_way_en,
300      Mux(
301        s1_need_replacement,
302        s1_repl_way_en,
303        s1_tag_match_way
304      )
305    )
306  )
307  assert(!RegNext(s1_fire && PopCount(s1_way_en) > 1.U))
308  val s1_tag = Mux(
309    s1_req.replace,
310    get_tag(s1_req.addr),
311    Mux(
312      s1_req.miss,
313      s1_miss_tag,
314      Mux(s1_need_replacement, s1_repl_tag, s1_hit_tag)
315    )
316  )
317  val s1_coh = Mux(
318    s1_req.replace,
319    Mux1H(s1_req.replace_way_en, meta_resp.map(ClientMetadata(_))),
320    Mux(
321      s1_req.miss,
322      s1_miss_coh,
323      Mux(s1_need_replacement, s1_repl_coh, s1_hit_coh)
324    )
325  )
326
327  val s1_has_permission = s1_hit_coh.onAccess(s1_req.cmd)._1
328  val s1_hit = s1_tag_match && s1_has_permission
329  val s1_pregen_can_go_to_mq = !s1_req.replace && !s1_req.probe && !s1_req.miss && (s1_req.isStore || s1_req.isAMO) && !s1_hit
330
331  // s2: select data, return resp if this is a store miss
332  val s2_valid = RegInit(false.B)
333  val s2_req = RegEnable(s1_req, s1_fire)
334  val s2_tag_match = RegEnable(s1_tag_match, s1_fire)
335  val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_fire)
336  val s2_hit_coh = RegEnable(s1_hit_coh, s1_fire)
337  val (s2_has_permission, _, s2_new_hit_coh) = s2_hit_coh.onAccess(s2_req.cmd)
338
339  val s2_repl_tag = RegEnable(s1_repl_tag, s1_fire)
340  val s2_repl_coh = RegEnable(s1_repl_coh, s1_fire)
341  val s2_repl_way_en = RegEnable(s1_repl_way_en, s1_fire)
342  val s2_need_replacement = RegEnable(s1_need_replacement, s1_fire)
343  val s2_need_data = RegEnable(s1_need_data, s1_fire)
344  val s2_need_tag = RegEnable(s1_need_tag, s1_fire)
345  val s2_encTag = RegEnable(s1_encTag, s1_fire)
346  val s2_idx = get_idx(s2_req.vaddr)
347
348  // duplicate regs to reduce fanout
349  val s2_valid_dup = RegInit(VecInit(Seq.fill(8)(false.B)))
350  val s2_valid_dup_for_status = RegInit(VecInit(Seq.fill(nDupStatus)(false.B)))
351  val s2_req_vaddr_dup_for_miss_req = RegEnable(s1_req.vaddr, s1_fire)
352  val s2_idx_dup_for_status = RegEnable(get_idx(s1_req.vaddr), s1_fire)
353  val s2_idx_dup_for_replace_access = RegEnable(get_idx(s1_req.vaddr), s1_fire)
354
355  val s2_req_replace_dup_1,
356      s2_req_replace_dup_2 = RegEnable(s1_req.replace, s1_fire)
357
358  val s2_can_go_to_mq_dup = (0 until 3).map(_ => RegEnable(s1_pregen_can_go_to_mq, s1_fire))
359
360  val s2_way_en = RegEnable(s1_way_en, s1_fire)
361  val s2_tag = RegEnable(s1_tag, s1_fire)
362  val s2_coh = RegEnable(s1_coh, s1_fire)
363  val s2_banked_store_wmask = RegEnable(s1_banked_store_wmask, s1_fire)
364  val s2_flag_error = RegEnable(s1_flag_error, s1_fire)
365  val s2_tag_error = dcacheParameters.tagCode.decode(s2_encTag).error && s2_need_tag
366  val s2_l2_error = s2_req.error
367  val s2_error = s2_flag_error || s2_tag_error || s2_l2_error // data_error not included
368
369  val s2_may_report_data_error = s2_need_data && s2_coh.state =/= ClientStates.Nothing
370
371  val s2_hit = s2_tag_match && s2_has_permission
372  val s2_amo_hit = s2_hit && !s2_req.probe && !s2_req.miss && s2_req.isAMO
373  val s2_store_hit = s2_hit && !s2_req.probe && !s2_req.miss && s2_req.isStore
374
375  s2_s0_set_conlict := s2_valid_dup(0) && s0_idx === s2_idx
376  s2_s0_set_conlict_store := s2_valid_dup(1) && store_idx === s2_idx
377
378  // For a store req, it either hits and goes to s3, or miss and enter miss queue immediately
379  val s2_can_go_to_s3 = (s2_req_replace_dup_1 || s2_req.probe || s2_req.miss || (s2_req.isStore || s2_req.isAMO) && s2_hit) && s3_ready
380  val s2_can_go_to_mq = RegEnable(s1_pregen_can_go_to_mq, s1_fire)
381  assert(RegNext(!(s2_valid && s2_can_go_to_s3 && s2_can_go_to_mq)))
382  val s2_can_go = s2_can_go_to_s3 || s2_can_go_to_mq
383  val s2_fire = s2_valid && s2_can_go
384  val s2_fire_to_s3 = s2_valid_dup(2) && s2_can_go_to_s3
385  when (s1_fire) {
386    s2_valid := true.B
387    s2_valid_dup.foreach(_ := true.B)
388    s2_valid_dup_for_status.foreach(_ := true.B)
389  }.elsewhen (s2_fire) {
390    s2_valid := false.B
391    s2_valid_dup.foreach(_ := false.B)
392    s2_valid_dup_for_status.foreach(_ := false.B)
393  }
394  s2_ready := !s2_valid_dup(3) || s2_can_go
395  val replay = !io.miss_req.ready
396
397  val data_resp = Wire(io.data_resp.cloneType)
398  data_resp := Mux(RegNext(s1_fire), io.data_resp, RegNext(data_resp))
399  val s2_store_data_merged = Wire(Vec(DCacheBanks, UInt(DCacheSRAMRowBits.W)))
400
401  def mergePutData(old_data: UInt, new_data: UInt, wmask: UInt): UInt = {
402    val full_wmask = FillInterleaved(8, wmask)
403    ((~full_wmask & old_data) | (full_wmask & new_data))
404  }
405
406  val s2_data = WireInit(VecInit((0 until DCacheBanks).map(i => {
407    data_resp(i).raw_data
408  })))
409
410  for (i <- 0 until DCacheBanks) {
411    val old_data = s2_data(i)
412    val new_data = get_data_of_bank(i, s2_req.store_data)
413    // for amo hit, we should use read out SRAM data
414    // do not merge with store data
415    val wmask = Mux(s2_amo_hit, 0.U(wordBytes.W), get_mask_of_bank(i, s2_req.store_mask))
416    s2_store_data_merged(i) := mergePutData(old_data, new_data, wmask)
417  }
418
419  val s2_data_word = s2_store_data_merged(s2_req.word_idx)
420
421  // s3: write data, meta and tag
422  val s3_valid = RegInit(false.B)
423  val s3_req = RegEnable(s2_req, s2_fire_to_s3)
424  // val s3_idx = get_idx(s3_req.vaddr)
425  val s3_tag = RegEnable(s2_tag, s2_fire_to_s3)
426  val s3_tag_match = RegEnable(s2_tag_match, s2_fire_to_s3)
427  val s3_coh = RegEnable(s2_coh, s2_fire_to_s3)
428  val s3_hit = RegEnable(s2_hit, s2_fire_to_s3)
429  val s3_amo_hit = RegEnable(s2_amo_hit, s2_fire_to_s3)
430  val s3_store_hit = RegEnable(s2_store_hit, s2_fire_to_s3)
431  val s3_hit_coh = RegEnable(s2_hit_coh, s2_fire_to_s3)
432  val s3_new_hit_coh = RegEnable(s2_new_hit_coh, s2_fire_to_s3)
433  val s3_way_en = RegEnable(s2_way_en, s2_fire_to_s3)
434  val s3_banked_store_wmask = RegEnable(s2_banked_store_wmask, s2_fire_to_s3)
435  val s3_store_data_merged = RegEnable(s2_store_data_merged, s2_fire_to_s3)
436  val s3_data_word = RegEnable(s2_data_word, s2_fire_to_s3)
437  val s3_data = RegEnable(s2_data, s2_fire_to_s3)
438  val s3_l2_error = s3_req.error
439  // data_error will be reported by data array 1 cycle after data read resp
440  val s3_data_error = Wire(Bool())
441  s3_data_error := Mux(RegNext(RegNext(s1_fire)), // ecc check result is generated 2 cycle after read req
442    io.readline_error_delayed && RegNext(s2_may_report_data_error),
443    RegNext(s3_data_error) // do not update s3_data_error if !s1_fire
444  )
445  // error signal for amo inst
446  // s3_error = s3_flag_error || s3_tag_error || s3_l2_error || s3_data_error
447  val s3_error = RegEnable(s2_error, s2_fire_to_s3) || s3_data_error
448  val (_, _, probe_new_coh) = s3_coh.onProbe(s3_req.probe_param)
449  val s3_need_replacement = RegEnable(s2_need_replacement, s2_fire_to_s3)
450
451
452  // duplicate regs to reduce fanout
453  val s3_valid_dup = RegInit(VecInit(Seq.fill(14)(false.B)))
454  val s3_valid_dup_for_status = RegInit(VecInit(Seq.fill(nDupStatus)(false.B)))
455  val s3_way_en_dup = (0 until 4).map(_ => RegEnable(s2_way_en, s2_fire_to_s3))
456  val s3_coh_dup = (0 until 6).map(_ => RegEnable(s2_coh, s2_fire_to_s3))
457  val s3_tag_match_dup = RegEnable(s2_tag_match, s2_fire_to_s3)
458
459  val s3_req_vaddr_dup_for_wb,
460      s3_req_vaddr_dup_for_data_write = RegEnable(s2_req.vaddr, s2_fire_to_s3)
461
462  val s3_idx_dup = (0 until 6).map(_ => RegEnable(get_idx(s2_req.vaddr), s2_fire_to_s3))
463
464  val s3_req_replace_dup = (0 until 8).map(_ => RegEnable(s2_req.replace, s2_fire_to_s3))
465  val s3_req_cmd_dup = (0 until 6).map(_ => RegEnable(s2_req.cmd, s2_fire_to_s3))
466  val s3_req_source_dup_1, s3_req_source_dup_2 = RegEnable(s2_req.source, s2_fire_to_s3)
467  val s3_req_addr_dup = (0 until 5).map(_ => RegEnable(s2_req.addr, s2_fire_to_s3))
468  val s3_req_probe_dup = (0 until 10).map(_ => RegEnable(s2_req.probe, s2_fire_to_s3))
469  val s3_req_miss_dup = (0 until 10).map(_ => RegEnable(s2_req.miss, s2_fire_to_s3))
470  val s3_req_word_idx_dup = (0 until DCacheBanks).map(_ => RegEnable(s2_req.word_idx, s2_fire_to_s3))
471
472  val s3_need_replacement_dup = RegEnable(s2_need_replacement, s2_fire_to_s3)
473
474  val s3_s_amoalu_dup = RegInit(VecInit(Seq.fill(3)(false.B)))
475
476  val s3_hit_coh_dup = RegEnable(s2_hit_coh, s2_fire_to_s3)
477  val s3_new_hit_coh_dup = (0 until 2).map(_ => RegEnable(s2_new_hit_coh, s2_fire_to_s3))
478  val s3_amo_hit_dup = RegEnable(s2_amo_hit, s2_fire_to_s3)
479  val s3_store_hit_dup = (0 until 2).map(_ => RegEnable(s2_store_hit, s2_fire_to_s3))
480
481  val lrsc_count_dup = RegInit(VecInit(Seq.fill(3)(0.U(log2Ceil(LRSCCycles).W))))
482  val lrsc_valid_dup = lrsc_count_dup.map { case cnt => cnt > LRSCBackOff.U }
483  val lrsc_addr_dup = Reg(UInt())
484
485  val s3_req_probe_param_dup = RegEnable(s2_req.probe_param, s2_fire_to_s3)
486  val (_, probe_shrink_param, _) = s3_coh.onProbe(s3_req_probe_param_dup)
487
488
489  val miss_update_meta = s3_req.miss
490  val probe_update_meta = s3_req_probe_dup(0) && s3_tag_match_dup && s3_coh_dup(0) =/= probe_new_coh
491  val store_update_meta = s3_req.isStore && !s3_req_probe_dup(1) && s3_hit_coh =/= s3_new_hit_coh_dup(0)
492  val amo_update_meta = s3_req.isAMO && !s3_req_probe_dup(2) && s3_hit_coh_dup =/= s3_new_hit_coh_dup(1)
493  val amo_wait_amoalu = s3_req.isAMO && s3_req_cmd_dup(0) =/= M_XLR && s3_req_cmd_dup(1) =/= M_XSC
494  val update_meta = (miss_update_meta || probe_update_meta || store_update_meta || amo_update_meta) && !s3_req_replace_dup(0)
495
496  def missCohGen(cmd: UInt, param: UInt, dirty: Bool) = {
497    val c = categorize(cmd)
498    MuxLookup(Cat(c, param, dirty), Nothing, Seq(
499      //(effect param) -> (next)
500      Cat(rd, toB, false.B)  -> Branch,
501      Cat(rd, toB, true.B)   -> Branch,
502      Cat(rd, toT, false.B)  -> Trunk,
503      Cat(rd, toT, true.B)   -> Dirty,
504      Cat(wi, toT, false.B)  -> Trunk,
505      Cat(wi, toT, true.B)   -> Dirty,
506      Cat(wr, toT, false.B)  -> Dirty,
507      Cat(wr, toT, true.B)   -> Dirty))
508  }
509  val miss_new_coh = ClientMetadata(missCohGen(s3_req_cmd_dup(2), s3_req.miss_param, s3_req.miss_dirty))
510
511  // LR, SC and AMO
512  val debug_sc_fail_addr = RegInit(0.U)
513  val debug_sc_fail_cnt  = RegInit(0.U(8.W))
514
515  val lrsc_count = RegInit(0.U(log2Ceil(LRSCCycles).W))
516  // val lrsc_valid = lrsc_count > LRSCBackOff.U
517  val lrsc_addr  = Reg(UInt())
518  val s3_lr = !s3_req_probe_dup(3) && s3_req.isAMO && s3_req_cmd_dup(3) === M_XLR
519  val s3_sc = !s3_req_probe_dup(4) && s3_req.isAMO && s3_req_cmd_dup(4) === M_XSC
520  val s3_lrsc_addr_match = lrsc_valid_dup(0) && lrsc_addr === get_block_addr(s3_req.addr)
521  val s3_sc_fail = s3_sc && !s3_lrsc_addr_match
522  val s3_sc_resp = Mux(s3_sc_fail, 1.U, 0.U)
523
524  val s3_can_do_amo = (s3_req_miss_dup(0) && !s3_req_probe_dup(5) && s3_req.isAMO) || s3_amo_hit
525  val s3_can_do_amo_write = s3_can_do_amo && isWrite(s3_req_cmd_dup(5)) && !s3_sc_fail
526
527  when (s3_valid_dup(0) && (s3_lr || s3_sc)) {
528    when (s3_can_do_amo && s3_lr) {
529      lrsc_count := (LRSCCycles - 1).U
530      lrsc_count_dup.foreach(_ := (LRSCCycles - 1).U)
531      lrsc_addr := get_block_addr(s3_req_addr_dup(0))
532      lrsc_addr_dup := get_block_addr(s3_req_addr_dup(0))
533    } .otherwise {
534      lrsc_count := 0.U
535      lrsc_count_dup.foreach(_ := 0.U)
536    }
537  }.elsewhen (io.invalid_resv_set) {
538    // when we release this block,
539    // we invalidate this reservation set
540    lrsc_count := 0.U
541    lrsc_count_dup.foreach(_ := 0.U)
542  }.elsewhen (lrsc_count > 0.U) {
543    lrsc_count := lrsc_count - 1.U
544    lrsc_count_dup.foreach({case cnt =>
545      cnt := cnt - 1.U
546    })
547  }
548
549  io.lrsc_locked_block.valid := lrsc_valid_dup(1)
550  io.lrsc_locked_block.bits  := lrsc_addr_dup
551  io.block_lr := RegNext(lrsc_count > 0.U)
552
553  // When we update update_resv_set, block all probe req in the next cycle
554  // It should give Probe reservation set addr compare an independent cycle,
555  // which will lead to better timing
556  io.update_resv_set := s3_valid_dup(1) && s3_lr && s3_can_do_amo
557
558  when (s3_valid_dup(2)) {
559    when (s3_req_addr_dup(1) === debug_sc_fail_addr) {
560      when (s3_sc_fail) {
561        debug_sc_fail_cnt := debug_sc_fail_cnt + 1.U
562      } .elsewhen (s3_sc) {
563        debug_sc_fail_cnt := 0.U
564      }
565    } .otherwise {
566      when (s3_sc_fail) {
567        debug_sc_fail_addr := s3_req_addr_dup(2)
568        debug_sc_fail_cnt  := 1.U
569        XSWarn(s3_sc_fail === 100.U, p"L1DCache failed too many SCs in a row 0x${Hexadecimal(debug_sc_fail_addr)}, check if sth went wrong\n")
570      }
571    }
572  }
573  // assert(debug_sc_fail_cnt < 100.U, "L1DCache failed too many SCs in a row")
574
575  val banked_amo_wmask = UIntToOH(s3_req.word_idx)
576  val update_data = s3_req_miss_dup(2) || s3_store_hit_dup(0) || s3_can_do_amo_write
577
578  // generate write data
579  // AMO hits
580  val s3_s_amoalu = RegInit(false.B)
581  val do_amoalu = amo_wait_amoalu && s3_valid_dup(3) && !s3_s_amoalu
582  val amoalu   = Module(new AMOALU(wordBits))
583  amoalu.io.mask := s3_req.amo_mask
584  amoalu.io.cmd  := s3_req.cmd
585  amoalu.io.lhs  := s3_data_word
586  amoalu.io.rhs  := s3_req.amo_data
587
588  // merge amo write data
589//  val amo_bitmask = FillInterleaved(8, s3_req.amo_mask)
590  val s3_amo_data_merged = Wire(Vec(DCacheBanks, UInt(DCacheSRAMRowBits.W)))
591  val s3_sc_data_merged = Wire(Vec(DCacheBanks, UInt(DCacheSRAMRowBits.W)))
592  for (i <- 0 until DCacheBanks) {
593    val old_data = s3_store_data_merged(i)
594    val new_data = amoalu.io.out
595    val wmask = Mux(
596      s3_req_word_idx_dup(i) === i.U,
597      ~0.U(wordBytes.W),
598      0.U(wordBytes.W)
599    )
600    s3_amo_data_merged(i) := mergePutData(old_data, new_data, wmask)
601    s3_sc_data_merged(i) := mergePutData(old_data, s3_req.amo_data,
602      Mux(s3_req_word_idx_dup(i) === i.U && !s3_sc_fail, s3_req.amo_mask, 0.U(wordBytes.W))
603    )
604  }
605  val s3_amo_data_merged_reg = RegEnable(s3_amo_data_merged, do_amoalu)
606  when(do_amoalu){
607    s3_s_amoalu := true.B
608    s3_s_amoalu_dup.foreach(_ := true.B)
609  }
610
611  val miss_wb = s3_req_miss_dup(3) && s3_need_replacement && s3_coh_dup(1).state =/= ClientStates.Nothing
612  val miss_wb_dup = s3_req_miss_dup(3) && s3_need_replacement_dup && s3_coh_dup(1).state =/= ClientStates.Nothing
613  val probe_wb = s3_req.probe
614  val replace_wb = s3_req.replace
615  val need_wb = miss_wb_dup || probe_wb || replace_wb
616
617  val (_, miss_shrink_param, _) = s3_coh_dup(2).onCacheControl(M_FLUSH)
618  val writeback_param = Mux(probe_wb, probe_shrink_param, miss_shrink_param)
619  val writeback_data = if (dcacheParameters.alwaysReleaseData) {
620    s3_tag_match && s3_req_probe_dup(6) && s3_req.probe_need_data ||
621      s3_coh_dup(3) === ClientStates.Dirty || (miss_wb || replace_wb) && s3_coh_dup(3).state =/= ClientStates.Nothing
622  } else {
623    s3_tag_match && s3_req_probe_dup(6) && s3_req.probe_need_data || s3_coh_dup(3) === ClientStates.Dirty
624  }
625
626  val s3_probe_can_go = s3_req_probe_dup(7) && io.wb.ready && (io.meta_write.ready || !probe_update_meta)
627  val s3_store_can_go = s3_req_source_dup_1 === STORE_SOURCE.U && !s3_req_probe_dup(8) && (io.meta_write.ready || !store_update_meta) && (io.data_write.ready || !update_data)
628  val s3_amo_can_go = s3_amo_hit_dup && (io.meta_write.ready || !amo_update_meta) && (io.data_write.ready || !update_data) && (s3_s_amoalu_dup(0) || !amo_wait_amoalu)
629  val s3_miss_can_go = s3_req_miss_dup(4) &&
630    (io.meta_write.ready || !amo_update_meta) &&
631    (io.data_write.ready || !update_data) &&
632    (s3_s_amoalu_dup(1) || !amo_wait_amoalu) &&
633    io.tag_write.ready &&
634    io.wb.ready
635  val s3_replace_nothing = s3_req_replace_dup(1) && s3_coh_dup(4).state === ClientStates.Nothing
636  val s3_replace_can_go = s3_req_replace_dup(2) && (s3_replace_nothing || io.wb.ready)
637  val s3_can_go = s3_probe_can_go || s3_store_can_go || s3_amo_can_go || s3_miss_can_go || s3_replace_can_go
638  val s3_update_data_cango = s3_store_can_go || s3_amo_can_go || s3_miss_can_go // used to speed up data_write gen
639
640  // ---------------- duplicate regs for meta_write.valid to solve fanout ----------------
641  val s3_req_miss_dup_for_meta_w_valid = RegEnable(s2_req.miss, s2_fire_to_s3)
642  val s3_req_probe_dup_for_meta_w_valid = RegEnable(s2_req.probe, s2_fire_to_s3)
643  val s3_tag_match_dup_for_meta_w_valid = RegEnable(s2_tag_match, s2_fire_to_s3)
644  val s3_coh_dup_for_meta_w_valid = RegEnable(s2_coh, s2_fire_to_s3)
645  val s3_req_probe_param_dup_for_meta_w_valid = RegEnable(s2_req.probe_param, s2_fire_to_s3)
646  val (_, _, probe_new_coh_dup_for_meta_w_valid) = s3_coh_dup_for_meta_w_valid.onProbe(s3_req_probe_param_dup_for_meta_w_valid)
647  val s3_req_source_dup_for_meta_w_valid = RegEnable(s2_req.source, s2_fire_to_s3)
648  val s3_req_cmd_dup_for_meta_w_valid = RegEnable(s2_req.cmd, s2_fire_to_s3)
649  val s3_req_replace_dup_for_meta_w_valid = RegEnable(s2_req.replace, s2_fire_to_s3)
650  val s3_hit_coh_dup_for_meta_w_valid = RegEnable(s2_hit_coh, s2_fire_to_s3)
651  val s3_new_hit_coh_dup_for_meta_w_valid = RegEnable(s2_new_hit_coh, s2_fire_to_s3)
652
653  val miss_update_meta_dup_for_meta_w_valid = s3_req_miss_dup_for_meta_w_valid
654  val probe_update_meta_dup_for_meta_w_valid = s3_req_probe_dup_for_meta_w_valid && s3_tag_match_dup_for_meta_w_valid && s3_coh_dup_for_meta_w_valid =/= probe_new_coh_dup_for_meta_w_valid
655  val store_update_meta_dup_for_meta_w_valid = s3_req_source_dup_for_meta_w_valid === STORE_SOURCE.U &&
656    !s3_req_probe_dup_for_meta_w_valid &&
657    s3_hit_coh_dup_for_meta_w_valid =/= s3_new_hit_coh_dup_for_meta_w_valid
658  val amo_update_meta_dup_for_meta_w_valid = s3_req_source_dup_for_meta_w_valid === AMO_SOURCE.U &&
659    !s3_req_probe_dup_for_meta_w_valid &&
660    s3_hit_coh_dup_for_meta_w_valid =/= s3_new_hit_coh_dup_for_meta_w_valid
661  val update_meta_dup_for_meta_w_valid = (
662    miss_update_meta_dup_for_meta_w_valid ||
663    probe_update_meta_dup_for_meta_w_valid ||
664    store_update_meta_dup_for_meta_w_valid ||
665    amo_update_meta_dup_for_meta_w_valid
666  ) && !s3_req_replace_dup_for_meta_w_valid
667
668  val s3_valid_dup_for_meta_w_valid = RegInit(false.B)
669  val s3_amo_hit_dup_for_meta_w_valid = RegEnable(s2_amo_hit, s2_fire_to_s3)
670  val s3_s_amoalu_dup_for_meta_w_valid = RegInit(false.B)
671  val amo_wait_amoalu_dup_for_meta_w_valid = s3_req_source_dup_for_meta_w_valid === AMO_SOURCE.U &&
672    s3_req_cmd_dup_for_meta_w_valid =/= M_XLR &&
673    s3_req_cmd_dup_for_meta_w_valid =/= M_XSC
674  val do_amoalu_dup_for_meta_w_valid = amo_wait_amoalu_dup_for_meta_w_valid && s3_valid_dup_for_meta_w_valid && !s3_s_amoalu_dup_for_meta_w_valid
675
676  val s3_store_hit_dup_for_meta_w_valid = RegEnable(s2_store_hit, s2_fire_to_s3)
677  val s3_req_addr_dup_for_meta_w_valid = RegEnable(s2_req.addr, s2_fire_to_s3)
678  val s3_can_do_amo_dup_for_meta_w_valid = (s3_req_miss_dup_for_meta_w_valid && !s3_req_probe_dup_for_meta_w_valid && s3_req_source_dup_for_meta_w_valid === AMO_SOURCE.U) ||
679    s3_amo_hit_dup_for_meta_w_valid
680
681  val s3_lr_dup_for_meta_w_valid = !s3_req_probe_dup_for_meta_w_valid && s3_req_source_dup_for_meta_w_valid === AMO_SOURCE.U && s3_req_cmd_dup_for_meta_w_valid === M_XLR
682  val s3_sc_dup_for_meta_w_valid = !s3_req_probe_dup_for_meta_w_valid && s3_req_source_dup_for_meta_w_valid === AMO_SOURCE.U && s3_req_cmd_dup_for_meta_w_valid === M_XSC
683  val lrsc_addr_dup_for_meta_w_valid = Reg(UInt())
684  val lrsc_count_dup_for_meta_w_valid = RegInit(0.U(log2Ceil(LRSCCycles).W))
685
686  when (s3_valid_dup_for_meta_w_valid && (s3_lr_dup_for_meta_w_valid || s3_sc_dup_for_meta_w_valid)) {
687    when (s3_can_do_amo_dup_for_meta_w_valid && s3_lr_dup_for_meta_w_valid) {
688      lrsc_count_dup_for_meta_w_valid := (LRSCCycles - 1).U
689      lrsc_addr_dup_for_meta_w_valid := get_block_addr(s3_req_addr_dup_for_meta_w_valid)
690    }.otherwise {
691      lrsc_count_dup_for_meta_w_valid := 0.U
692    }
693  }.elsewhen (io.invalid_resv_set) {
694    lrsc_count_dup_for_meta_w_valid := 0.U
695  }.elsewhen (lrsc_count_dup_for_meta_w_valid > 0.U) {
696    lrsc_count_dup_for_meta_w_valid := lrsc_count_dup_for_meta_w_valid - 1.U
697  }
698
699  val lrsc_valid_dup_for_meta_w_valid = lrsc_count_dup_for_meta_w_valid > LRSCBackOff.U
700  val s3_lrsc_addr_match_dup_for_meta_w_valid = lrsc_valid_dup_for_meta_w_valid && lrsc_addr_dup_for_meta_w_valid === get_block_addr(s3_req_addr_dup_for_meta_w_valid)
701  val s3_sc_fail_dup_for_meta_w_valid = s3_sc_dup_for_meta_w_valid && !s3_lrsc_addr_match_dup_for_meta_w_valid
702  val s3_can_do_amo_write_dup_for_meta_w_valid = s3_can_do_amo_dup_for_meta_w_valid && isWrite(s3_req_cmd_dup_for_meta_w_valid) && !s3_sc_fail_dup_for_meta_w_valid
703  val update_data_dup_for_meta_w_valid = s3_req_miss_dup_for_meta_w_valid || s3_store_hit_dup_for_meta_w_valid || s3_can_do_amo_write_dup_for_meta_w_valid
704
705  val s3_probe_can_go_dup_for_meta_w_valid = s3_req_probe_dup_for_meta_w_valid &&
706    io.wb_ready_dup(metaWritePort) &&
707    (io.meta_write.ready || !probe_update_meta_dup_for_meta_w_valid)
708  val s3_store_can_go_dup_for_meta_w_valid = s3_req_source_dup_for_meta_w_valid === STORE_SOURCE.U && !s3_req_probe_dup_for_meta_w_valid &&
709    (io.meta_write.ready || !store_update_meta_dup_for_meta_w_valid) &&
710    (io.data_write_ready_dup(metaWritePort) || !update_data_dup_for_meta_w_valid)
711  val s3_amo_can_go_dup_for_meta_w_valid = s3_amo_hit_dup_for_meta_w_valid &&
712    (io.meta_write.ready || !amo_update_meta_dup_for_meta_w_valid) &&
713    (io.data_write_ready_dup(metaWritePort) || !update_data_dup_for_meta_w_valid) &&
714    (s3_s_amoalu_dup_for_meta_w_valid || !amo_wait_amoalu_dup_for_meta_w_valid)
715  val s3_miss_can_go_dup_for_meta_w_valid = s3_req_miss_dup_for_meta_w_valid &&
716    (io.meta_write.ready || !amo_update_meta_dup_for_meta_w_valid) &&
717    (io.data_write_ready_dup(metaWritePort) || !update_data_dup_for_meta_w_valid) &&
718    (s3_s_amoalu_dup_for_meta_w_valid || !amo_wait_amoalu_dup_for_meta_w_valid) &&
719    io.tag_write_ready_dup(metaWritePort) &&
720    io.wb_ready_dup(metaWritePort)
721  val s3_replace_can_go_dup_for_meta_w_valid = s3_req_replace_dup_for_meta_w_valid &&
722    (s3_coh_dup_for_meta_w_valid.state === ClientStates.Nothing || io.wb_ready_dup(metaWritePort))
723  val s3_can_go_dup_for_meta_w_valid = s3_probe_can_go_dup_for_meta_w_valid ||
724    s3_store_can_go_dup_for_meta_w_valid ||
725    s3_amo_can_go_dup_for_meta_w_valid ||
726    s3_miss_can_go_dup_for_meta_w_valid ||
727    s3_replace_can_go_dup_for_meta_w_valid
728
729  val s3_fire_dup_for_meta_w_valid = s3_valid_dup_for_meta_w_valid && s3_can_go_dup_for_meta_w_valid
730  when (do_amoalu_dup_for_meta_w_valid) { s3_s_amoalu_dup_for_meta_w_valid := true.B }
731  when (s3_fire_dup_for_meta_w_valid) { s3_s_amoalu_dup_for_meta_w_valid := false.B }
732
733  val new_coh = Mux(
734    miss_update_meta_dup_for_meta_w_valid,
735    miss_new_coh,
736    Mux(
737      probe_update_meta,
738      probe_new_coh_dup_for_meta_w_valid,
739      Mux(
740        store_update_meta_dup_for_meta_w_valid || amo_update_meta_dup_for_meta_w_valid,
741        s3_new_hit_coh_dup_for_meta_w_valid,
742        ClientMetadata.onReset
743      )
744    )
745  )
746
747  when (s2_fire_to_s3) { s3_valid_dup_for_meta_w_valid := true.B }
748  .elsewhen (s3_fire_dup_for_meta_w_valid) { s3_valid_dup_for_meta_w_valid := false.B }
749  // -------------------------------------------------------------------------------------
750
751  // ---------------- duplicate regs for err_write.valid to solve fanout -----------------
752  val s3_req_miss_dup_for_err_w_valid = RegEnable(s2_req.miss, s2_fire_to_s3)
753  val s3_req_probe_dup_for_err_w_valid = RegEnable(s2_req.probe, s2_fire_to_s3)
754  val s3_tag_match_dup_for_err_w_valid = RegEnable(s2_tag_match, s2_fire_to_s3)
755  val s3_coh_dup_for_err_w_valid = RegEnable(s2_coh, s2_fire_to_s3)
756  val s3_req_probe_param_dup_for_err_w_valid = RegEnable(s2_req.probe_param, s2_fire_to_s3)
757  val (_, _, probe_new_coh_dup_for_err_w_valid) = s3_coh_dup_for_err_w_valid.onProbe(s3_req_probe_param_dup_for_err_w_valid)
758  val s3_req_source_dup_for_err_w_valid = RegEnable(s2_req.source, s2_fire_to_s3)
759  val s3_req_cmd_dup_for_err_w_valid = RegEnable(s2_req.cmd, s2_fire_to_s3)
760  val s3_req_replace_dup_for_err_w_valid = RegEnable(s2_req.replace, s2_fire_to_s3)
761  val s3_hit_coh_dup_for_err_w_valid = RegEnable(s2_hit_coh, s2_fire_to_s3)
762  val s3_new_hit_coh_dup_for_err_w_valid = RegEnable(s2_new_hit_coh, s2_fire_to_s3)
763
764  val miss_update_meta_dup_for_err_w_valid = s3_req_miss_dup_for_err_w_valid
765  val probe_update_meta_dup_for_err_w_valid = s3_req_probe_dup_for_err_w_valid && s3_tag_match_dup_for_err_w_valid && s3_coh_dup_for_err_w_valid =/= probe_new_coh_dup_for_err_w_valid
766  val store_update_meta_dup_for_err_w_valid = s3_req_source_dup_for_err_w_valid === STORE_SOURCE.U &&
767    !s3_req_probe_dup_for_err_w_valid &&
768    s3_hit_coh_dup_for_err_w_valid =/= s3_new_hit_coh_dup_for_err_w_valid
769  val amo_update_meta_dup_for_err_w_valid = s3_req_source_dup_for_err_w_valid === AMO_SOURCE.U &&
770    !s3_req_probe_dup_for_err_w_valid &&
771    s3_hit_coh_dup_for_err_w_valid =/= s3_new_hit_coh_dup_for_err_w_valid
772  val update_meta_dup_for_err_w_valid = (
773    miss_update_meta_dup_for_err_w_valid ||
774    probe_update_meta_dup_for_err_w_valid ||
775    store_update_meta_dup_for_err_w_valid ||
776    amo_update_meta_dup_for_err_w_valid
777  ) && !s3_req_replace_dup_for_err_w_valid
778
779  val s3_valid_dup_for_err_w_valid = RegInit(false.B)
780  val s3_amo_hit_dup_for_err_w_valid = RegEnable(s2_amo_hit, s2_fire_to_s3)
781  val s3_s_amoalu_dup_for_err_w_valid = RegInit(false.B)
782  val amo_wait_amoalu_dup_for_err_w_valid = s3_req_source_dup_for_err_w_valid === AMO_SOURCE.U &&
783    s3_req_cmd_dup_for_err_w_valid =/= M_XLR &&
784    s3_req_cmd_dup_for_err_w_valid =/= M_XSC
785  val do_amoalu_dup_for_err_w_valid = amo_wait_amoalu_dup_for_err_w_valid && s3_valid_dup_for_err_w_valid && !s3_s_amoalu_dup_for_err_w_valid
786
787  val s3_store_hit_dup_for_err_w_valid = RegEnable(s2_store_hit, s2_fire_to_s3)
788  val s3_req_addr_dup_for_err_w_valid = RegEnable(s2_req.addr, s2_fire_to_s3)
789  val s3_can_do_amo_dup_for_err_w_valid = (s3_req_miss_dup_for_err_w_valid && !s3_req_probe_dup_for_err_w_valid && s3_req_source_dup_for_err_w_valid === AMO_SOURCE.U) ||
790    s3_amo_hit_dup_for_err_w_valid
791
792  val s3_lr_dup_for_err_w_valid = !s3_req_probe_dup_for_err_w_valid && s3_req_source_dup_for_err_w_valid === AMO_SOURCE.U && s3_req_cmd_dup_for_err_w_valid === M_XLR
793  val s3_sc_dup_for_err_w_valid = !s3_req_probe_dup_for_err_w_valid && s3_req_source_dup_for_err_w_valid === AMO_SOURCE.U && s3_req_cmd_dup_for_err_w_valid === M_XSC
794  val lrsc_addr_dup_for_err_w_valid = Reg(UInt())
795  val lrsc_count_dup_for_err_w_valid = RegInit(0.U(log2Ceil(LRSCCycles).W))
796
797  when (s3_valid_dup_for_err_w_valid && (s3_lr_dup_for_err_w_valid || s3_sc_dup_for_err_w_valid)) {
798    when (s3_can_do_amo_dup_for_err_w_valid && s3_lr_dup_for_err_w_valid) {
799      lrsc_count_dup_for_err_w_valid := (LRSCCycles - 1).U
800      lrsc_addr_dup_for_err_w_valid := get_block_addr(s3_req_addr_dup_for_err_w_valid)
801    }.otherwise {
802      lrsc_count_dup_for_err_w_valid := 0.U
803    }
804  }.elsewhen (io.invalid_resv_set) {
805    lrsc_count_dup_for_err_w_valid := 0.U
806  }.elsewhen (lrsc_count_dup_for_err_w_valid > 0.U) {
807    lrsc_count_dup_for_err_w_valid := lrsc_count_dup_for_err_w_valid - 1.U
808  }
809
810  val lrsc_valid_dup_for_err_w_valid = lrsc_count_dup_for_err_w_valid > LRSCBackOff.U
811  val s3_lrsc_addr_match_dup_for_err_w_valid = lrsc_valid_dup_for_err_w_valid && lrsc_addr_dup_for_err_w_valid === get_block_addr(s3_req_addr_dup_for_err_w_valid)
812  val s3_sc_fail_dup_for_err_w_valid = s3_sc_dup_for_err_w_valid && !s3_lrsc_addr_match_dup_for_err_w_valid
813  val s3_can_do_amo_write_dup_for_err_w_valid = s3_can_do_amo_dup_for_err_w_valid && isWrite(s3_req_cmd_dup_for_err_w_valid) && !s3_sc_fail_dup_for_err_w_valid
814  val update_data_dup_for_err_w_valid = s3_req_miss_dup_for_err_w_valid || s3_store_hit_dup_for_err_w_valid || s3_can_do_amo_write_dup_for_err_w_valid
815
816  val s3_probe_can_go_dup_for_err_w_valid = s3_req_probe_dup_for_err_w_valid &&
817    io.wb_ready_dup(errWritePort) &&
818    (io.meta_write.ready || !probe_update_meta_dup_for_err_w_valid)
819  val s3_store_can_go_dup_for_err_w_valid = s3_req_source_dup_for_err_w_valid === STORE_SOURCE.U && !s3_req_probe_dup_for_err_w_valid &&
820    (io.meta_write.ready || !store_update_meta_dup_for_err_w_valid) &&
821    (io.data_write_ready_dup(errWritePort) || !update_data_dup_for_err_w_valid)
822  val s3_amo_can_go_dup_for_err_w_valid = s3_amo_hit_dup_for_err_w_valid &&
823    (io.meta_write.ready || !amo_update_meta_dup_for_err_w_valid) &&
824    (io.data_write_ready_dup(errWritePort) || !update_data_dup_for_err_w_valid) &&
825    (s3_s_amoalu_dup_for_err_w_valid || !amo_wait_amoalu_dup_for_err_w_valid)
826  val s3_miss_can_go_dup_for_err_w_valid = s3_req_miss_dup_for_err_w_valid &&
827    (io.meta_write.ready || !amo_update_meta_dup_for_err_w_valid) &&
828    (io.data_write_ready_dup(errWritePort) || !update_data_dup_for_err_w_valid) &&
829    (s3_s_amoalu_dup_for_err_w_valid || !amo_wait_amoalu_dup_for_err_w_valid) &&
830    io.tag_write_ready_dup(errWritePort) &&
831    io.wb_ready_dup(errWritePort)
832  val s3_replace_can_go_dup_for_err_w_valid = s3_req_replace_dup_for_err_w_valid &&
833    (s3_coh_dup_for_err_w_valid.state === ClientStates.Nothing || io.wb_ready_dup(errWritePort))
834  val s3_can_go_dup_for_err_w_valid = s3_probe_can_go_dup_for_err_w_valid ||
835    s3_store_can_go_dup_for_err_w_valid ||
836    s3_amo_can_go_dup_for_err_w_valid ||
837    s3_miss_can_go_dup_for_err_w_valid ||
838    s3_replace_can_go_dup_for_err_w_valid
839
840  val s3_fire_dup_for_err_w_valid = s3_valid_dup_for_err_w_valid && s3_can_go_dup_for_err_w_valid
841  when (do_amoalu_dup_for_err_w_valid) { s3_s_amoalu_dup_for_err_w_valid := true.B }
842  when (s3_fire_dup_for_err_w_valid) { s3_s_amoalu_dup_for_err_w_valid := false.B }
843
844  when (s2_fire_to_s3) { s3_valid_dup_for_err_w_valid := true.B }
845  .elsewhen (s3_fire_dup_for_err_w_valid) { s3_valid_dup_for_err_w_valid := false.B }
846  // -------------------------------------------------------------------------------------
847  // ---------------- duplicate regs for tag_write.valid to solve fanout -----------------
848  val s3_req_miss_dup_for_tag_w_valid = RegEnable(s2_req.miss, s2_fire_to_s3)
849  val s3_req_probe_dup_for_tag_w_valid = RegEnable(s2_req.probe, s2_fire_to_s3)
850  val s3_tag_match_dup_for_tag_w_valid = RegEnable(s2_tag_match, s2_fire_to_s3)
851  val s3_coh_dup_for_tag_w_valid = RegEnable(s2_coh, s2_fire_to_s3)
852  val s3_req_probe_param_dup_for_tag_w_valid = RegEnable(s2_req.probe_param, s2_fire_to_s3)
853  val (_, _, probe_new_coh_dup_for_tag_w_valid) = s3_coh_dup_for_tag_w_valid.onProbe(s3_req_probe_param_dup_for_tag_w_valid)
854  val s3_req_source_dup_for_tag_w_valid = RegEnable(s2_req.source, s2_fire_to_s3)
855  val s3_req_cmd_dup_for_tag_w_valid = RegEnable(s2_req.cmd, s2_fire_to_s3)
856  val s3_req_replace_dup_for_tag_w_valid = RegEnable(s2_req.replace, s2_fire_to_s3)
857  val s3_hit_coh_dup_for_tag_w_valid = RegEnable(s2_hit_coh, s2_fire_to_s3)
858  val s3_new_hit_coh_dup_for_tag_w_valid = RegEnable(s2_new_hit_coh, s2_fire_to_s3)
859
860  val miss_update_meta_dup_for_tag_w_valid = s3_req_miss_dup_for_tag_w_valid
861  val probe_update_meta_dup_for_tag_w_valid = s3_req_probe_dup_for_tag_w_valid && s3_tag_match_dup_for_tag_w_valid && s3_coh_dup_for_tag_w_valid =/= probe_new_coh_dup_for_tag_w_valid
862  val store_update_meta_dup_for_tag_w_valid = s3_req_source_dup_for_tag_w_valid === STORE_SOURCE.U &&
863    !s3_req_probe_dup_for_tag_w_valid &&
864    s3_hit_coh_dup_for_tag_w_valid =/= s3_new_hit_coh_dup_for_tag_w_valid
865  val amo_update_meta_dup_for_tag_w_valid = s3_req_source_dup_for_tag_w_valid === AMO_SOURCE.U &&
866    !s3_req_probe_dup_for_tag_w_valid &&
867    s3_hit_coh_dup_for_tag_w_valid =/= s3_new_hit_coh_dup_for_tag_w_valid
868  val update_meta_dup_for_tag_w_valid = (
869    miss_update_meta_dup_for_tag_w_valid ||
870    probe_update_meta_dup_for_tag_w_valid ||
871    store_update_meta_dup_for_tag_w_valid ||
872    amo_update_meta_dup_for_tag_w_valid
873  ) && !s3_req_replace_dup_for_tag_w_valid
874
875  val s3_valid_dup_for_tag_w_valid = RegInit(false.B)
876  val s3_amo_hit_dup_for_tag_w_valid = RegEnable(s2_amo_hit, s2_fire_to_s3)
877  val s3_s_amoalu_dup_for_tag_w_valid = RegInit(false.B)
878  val amo_wait_amoalu_dup_for_tag_w_valid = s3_req_source_dup_for_tag_w_valid === AMO_SOURCE.U &&
879    s3_req_cmd_dup_for_tag_w_valid =/= M_XLR &&
880    s3_req_cmd_dup_for_tag_w_valid =/= M_XSC
881  val do_amoalu_dup_for_tag_w_valid = amo_wait_amoalu_dup_for_tag_w_valid && s3_valid_dup_for_tag_w_valid && !s3_s_amoalu_dup_for_tag_w_valid
882
883  val s3_store_hit_dup_for_tag_w_valid = RegEnable(s2_store_hit, s2_fire_to_s3)
884  val s3_req_addr_dup_for_tag_w_valid = RegEnable(s2_req.addr, s2_fire_to_s3)
885  val s3_can_do_amo_dup_for_tag_w_valid = (s3_req_miss_dup_for_tag_w_valid && !s3_req_probe_dup_for_tag_w_valid && s3_req_source_dup_for_tag_w_valid === AMO_SOURCE.U) ||
886    s3_amo_hit_dup_for_tag_w_valid
887
888  val s3_lr_dup_for_tag_w_valid = !s3_req_probe_dup_for_tag_w_valid && s3_req_source_dup_for_tag_w_valid === AMO_SOURCE.U && s3_req_cmd_dup_for_tag_w_valid === M_XLR
889  val s3_sc_dup_for_tag_w_valid = !s3_req_probe_dup_for_tag_w_valid && s3_req_source_dup_for_tag_w_valid === AMO_SOURCE.U && s3_req_cmd_dup_for_tag_w_valid === M_XSC
890  val lrsc_addr_dup_for_tag_w_valid = Reg(UInt())
891  val lrsc_count_dup_for_tag_w_valid = RegInit(0.U(log2Ceil(LRSCCycles).W))
892
893  when (s3_valid_dup_for_tag_w_valid && (s3_lr_dup_for_tag_w_valid || s3_sc_dup_for_tag_w_valid)) {
894    when (s3_can_do_amo_dup_for_tag_w_valid && s3_lr_dup_for_tag_w_valid) {
895      lrsc_count_dup_for_tag_w_valid := (LRSCCycles - 1).U
896      lrsc_addr_dup_for_tag_w_valid := get_block_addr(s3_req_addr_dup_for_tag_w_valid)
897    }.otherwise {
898      lrsc_count_dup_for_tag_w_valid := 0.U
899    }
900  }.elsewhen (io.invalid_resv_set) {
901    lrsc_count_dup_for_tag_w_valid := 0.U
902  }.elsewhen (lrsc_count_dup_for_tag_w_valid > 0.U) {
903    lrsc_count_dup_for_tag_w_valid := lrsc_count_dup_for_tag_w_valid - 1.U
904  }
905
906  val lrsc_valid_dup_for_tag_w_valid = lrsc_count_dup_for_tag_w_valid > LRSCBackOff.U
907  val s3_lrsc_addr_match_dup_for_tag_w_valid = lrsc_valid_dup_for_tag_w_valid && lrsc_addr_dup_for_tag_w_valid === get_block_addr(s3_req_addr_dup_for_tag_w_valid)
908  val s3_sc_fail_dup_for_tag_w_valid = s3_sc_dup_for_tag_w_valid && !s3_lrsc_addr_match_dup_for_tag_w_valid
909  val s3_can_do_amo_write_dup_for_tag_w_valid = s3_can_do_amo_dup_for_tag_w_valid && isWrite(s3_req_cmd_dup_for_tag_w_valid) && !s3_sc_fail_dup_for_tag_w_valid
910  val update_data_dup_for_tag_w_valid = s3_req_miss_dup_for_tag_w_valid || s3_store_hit_dup_for_tag_w_valid || s3_can_do_amo_write_dup_for_tag_w_valid
911
912  val s3_probe_can_go_dup_for_tag_w_valid = s3_req_probe_dup_for_tag_w_valid &&
913    io.wb_ready_dup(tagWritePort) &&
914    (io.meta_write.ready || !probe_update_meta_dup_for_tag_w_valid)
915  val s3_store_can_go_dup_for_tag_w_valid = s3_req_source_dup_for_tag_w_valid === STORE_SOURCE.U && !s3_req_probe_dup_for_tag_w_valid &&
916    (io.meta_write.ready || !store_update_meta_dup_for_tag_w_valid) &&
917    (io.data_write_ready_dup(tagWritePort) || !update_data_dup_for_tag_w_valid)
918  val s3_amo_can_go_dup_for_tag_w_valid = s3_amo_hit_dup_for_tag_w_valid &&
919    (io.meta_write.ready || !amo_update_meta_dup_for_tag_w_valid) &&
920    (io.data_write_ready_dup(tagWritePort) || !update_data_dup_for_tag_w_valid) &&
921    (s3_s_amoalu_dup_for_tag_w_valid || !amo_wait_amoalu_dup_for_tag_w_valid)
922  val s3_miss_can_go_dup_for_tag_w_valid = s3_req_miss_dup_for_tag_w_valid &&
923    (io.meta_write.ready || !amo_update_meta_dup_for_tag_w_valid) &&
924    (io.data_write_ready_dup(tagWritePort) || !update_data_dup_for_tag_w_valid) &&
925    (s3_s_amoalu_dup_for_tag_w_valid || !amo_wait_amoalu_dup_for_tag_w_valid) &&
926    io.tag_write_ready_dup(tagWritePort) &&
927    io.wb_ready_dup(tagWritePort)
928  val s3_replace_can_go_dup_for_tag_w_valid = s3_req_replace_dup_for_tag_w_valid &&
929    (s3_coh_dup_for_tag_w_valid.state === ClientStates.Nothing || io.wb_ready_dup(tagWritePort))
930  val s3_can_go_dup_for_tag_w_valid = s3_probe_can_go_dup_for_tag_w_valid ||
931    s3_store_can_go_dup_for_tag_w_valid ||
932    s3_amo_can_go_dup_for_tag_w_valid ||
933    s3_miss_can_go_dup_for_tag_w_valid ||
934    s3_replace_can_go_dup_for_tag_w_valid
935
936  val s3_fire_dup_for_tag_w_valid = s3_valid_dup_for_tag_w_valid && s3_can_go_dup_for_tag_w_valid
937  when (do_amoalu_dup_for_tag_w_valid) { s3_s_amoalu_dup_for_tag_w_valid := true.B }
938  when (s3_fire_dup_for_tag_w_valid) { s3_s_amoalu_dup_for_tag_w_valid := false.B }
939
940  when (s2_fire_to_s3) { s3_valid_dup_for_tag_w_valid := true.B }
941  .elsewhen (s3_fire_dup_for_tag_w_valid) { s3_valid_dup_for_tag_w_valid := false.B }
942  // -------------------------------------------------------------------------------------
943  // ---------------- duplicate regs for data_write.valid to solve fanout ----------------
944  val s3_req_miss_dup_for_data_w_valid = RegEnable(s2_req.miss, s2_fire_to_s3)
945  val s3_req_probe_dup_for_data_w_valid = RegEnable(s2_req.probe, s2_fire_to_s3)
946  val s3_tag_match_dup_for_data_w_valid = RegEnable(s2_tag_match, s2_fire_to_s3)
947  val s3_coh_dup_for_data_w_valid = RegEnable(s2_coh, s2_fire_to_s3)
948  val s3_req_probe_param_dup_for_data_w_valid = RegEnable(s2_req.probe_param, s2_fire_to_s3)
949  val (_, _, probe_new_coh_dup_for_data_w_valid) = s3_coh_dup_for_data_w_valid.onProbe(s3_req_probe_param_dup_for_data_w_valid)
950  val s3_req_source_dup_for_data_w_valid = RegEnable(s2_req.source, s2_fire_to_s3)
951  val s3_req_cmd_dup_for_data_w_valid = RegEnable(s2_req.cmd, s2_fire_to_s3)
952  val s3_req_replace_dup_for_data_w_valid = RegEnable(s2_req.replace, s2_fire_to_s3)
953  val s3_hit_coh_dup_for_data_w_valid = RegEnable(s2_hit_coh, s2_fire_to_s3)
954  val s3_new_hit_coh_dup_for_data_w_valid = RegEnable(s2_new_hit_coh, s2_fire_to_s3)
955
956  val miss_update_meta_dup_for_data_w_valid = s3_req_miss_dup_for_data_w_valid
957  val probe_update_meta_dup_for_data_w_valid = s3_req_probe_dup_for_data_w_valid && s3_tag_match_dup_for_data_w_valid && s3_coh_dup_for_data_w_valid =/= probe_new_coh_dup_for_data_w_valid
958  val store_update_meta_dup_for_data_w_valid = s3_req_source_dup_for_data_w_valid === STORE_SOURCE.U &&
959    !s3_req_probe_dup_for_data_w_valid &&
960    s3_hit_coh_dup_for_data_w_valid =/= s3_new_hit_coh_dup_for_data_w_valid
961  val amo_update_meta_dup_for_data_w_valid = s3_req_source_dup_for_data_w_valid === AMO_SOURCE.U &&
962    !s3_req_probe_dup_for_data_w_valid &&
963    s3_hit_coh_dup_for_data_w_valid =/= s3_new_hit_coh_dup_for_data_w_valid
964  val update_meta_dup_for_data_w_valid = (
965    miss_update_meta_dup_for_data_w_valid ||
966    probe_update_meta_dup_for_data_w_valid ||
967    store_update_meta_dup_for_data_w_valid ||
968    amo_update_meta_dup_for_data_w_valid
969  ) && !s3_req_replace_dup_for_data_w_valid
970
971  val s3_valid_dup_for_data_w_valid = RegInit(false.B)
972  val s3_amo_hit_dup_for_data_w_valid = RegEnable(s2_amo_hit, s2_fire_to_s3)
973  val s3_s_amoalu_dup_for_data_w_valid = RegInit(false.B)
974  val amo_wait_amoalu_dup_for_data_w_valid = s3_req_source_dup_for_data_w_valid === AMO_SOURCE.U &&
975    s3_req_cmd_dup_for_data_w_valid =/= M_XLR &&
976    s3_req_cmd_dup_for_data_w_valid =/= M_XSC
977  val do_amoalu_dup_for_data_w_valid = amo_wait_amoalu_dup_for_data_w_valid && s3_valid_dup_for_data_w_valid && !s3_s_amoalu_dup_for_data_w_valid
978
979  val s3_store_hit_dup_for_data_w_valid = RegEnable(s2_store_hit, s2_fire_to_s3)
980  val s3_req_addr_dup_for_data_w_valid = RegEnable(s2_req.addr, s2_fire_to_s3)
981  val s3_can_do_amo_dup_for_data_w_valid = (s3_req_miss_dup_for_data_w_valid && !s3_req_probe_dup_for_data_w_valid && s3_req_source_dup_for_data_w_valid === AMO_SOURCE.U) ||
982    s3_amo_hit_dup_for_data_w_valid
983
984  val s3_lr_dup_for_data_w_valid = !s3_req_probe_dup_for_data_w_valid && s3_req_source_dup_for_data_w_valid === AMO_SOURCE.U && s3_req_cmd_dup_for_data_w_valid === M_XLR
985  val s3_sc_dup_for_data_w_valid = !s3_req_probe_dup_for_data_w_valid && s3_req_source_dup_for_data_w_valid === AMO_SOURCE.U && s3_req_cmd_dup_for_data_w_valid === M_XSC
986  val lrsc_addr_dup_for_data_w_valid = Reg(UInt())
987  val lrsc_count_dup_for_data_w_valid = RegInit(0.U(log2Ceil(LRSCCycles).W))
988
989  when (s3_valid_dup_for_data_w_valid && (s3_lr_dup_for_data_w_valid || s3_sc_dup_for_data_w_valid)) {
990    when (s3_can_do_amo_dup_for_data_w_valid && s3_lr_dup_for_data_w_valid) {
991      lrsc_count_dup_for_data_w_valid := (LRSCCycles - 1).U
992      lrsc_addr_dup_for_data_w_valid := get_block_addr(s3_req_addr_dup_for_data_w_valid)
993    }.otherwise {
994      lrsc_count_dup_for_data_w_valid := 0.U
995    }
996  }.elsewhen (io.invalid_resv_set) {
997    lrsc_count_dup_for_data_w_valid := 0.U
998  }.elsewhen (lrsc_count_dup_for_data_w_valid > 0.U) {
999    lrsc_count_dup_for_data_w_valid := lrsc_count_dup_for_data_w_valid - 1.U
1000  }
1001
1002  val lrsc_valid_dup_for_data_w_valid = lrsc_count_dup_for_data_w_valid > LRSCBackOff.U
1003  val s3_lrsc_addr_match_dup_for_data_w_valid = lrsc_valid_dup_for_data_w_valid && lrsc_addr_dup_for_data_w_valid === get_block_addr(s3_req_addr_dup_for_data_w_valid)
1004  val s3_sc_fail_dup_for_data_w_valid = s3_sc_dup_for_data_w_valid && !s3_lrsc_addr_match_dup_for_data_w_valid
1005  val s3_can_do_amo_write_dup_for_data_w_valid = s3_can_do_amo_dup_for_data_w_valid && isWrite(s3_req_cmd_dup_for_data_w_valid) && !s3_sc_fail_dup_for_data_w_valid
1006  val update_data_dup_for_data_w_valid = s3_req_miss_dup_for_data_w_valid || s3_store_hit_dup_for_data_w_valid || s3_can_do_amo_write_dup_for_data_w_valid
1007
1008  val s3_probe_can_go_dup_for_data_w_valid = s3_req_probe_dup_for_data_w_valid &&
1009    io.wb_ready_dup(dataWritePort) &&
1010    (io.meta_write.ready || !probe_update_meta_dup_for_data_w_valid)
1011  val s3_store_can_go_dup_for_data_w_valid = s3_req_source_dup_for_data_w_valid === STORE_SOURCE.U && !s3_req_probe_dup_for_data_w_valid &&
1012    (io.meta_write.ready || !store_update_meta_dup_for_data_w_valid) &&
1013    (io.data_write_ready_dup(dataWritePort) || !update_data_dup_for_data_w_valid)
1014  val s3_amo_can_go_dup_for_data_w_valid = s3_amo_hit_dup_for_data_w_valid &&
1015    (io.meta_write.ready || !amo_update_meta_dup_for_data_w_valid) &&
1016    (io.data_write_ready_dup(dataWritePort) || !update_data_dup_for_data_w_valid) &&
1017    (s3_s_amoalu_dup_for_data_w_valid || !amo_wait_amoalu_dup_for_data_w_valid)
1018  val s3_miss_can_go_dup_for_data_w_valid = s3_req_miss_dup_for_data_w_valid &&
1019    (io.meta_write.ready || !amo_update_meta_dup_for_data_w_valid) &&
1020    (io.data_write_ready_dup(dataWritePort) || !update_data_dup_for_data_w_valid) &&
1021    (s3_s_amoalu_dup_for_data_w_valid || !amo_wait_amoalu_dup_for_data_w_valid) &&
1022    io.tag_write_ready_dup(dataWritePort) &&
1023    io.wb_ready_dup(dataWritePort)
1024  val s3_replace_can_go_dup_for_data_w_valid = s3_req_replace_dup_for_data_w_valid &&
1025    (s3_coh_dup_for_data_w_valid.state === ClientStates.Nothing || io.wb_ready_dup(dataWritePort))
1026  val s3_can_go_dup_for_data_w_valid = s3_probe_can_go_dup_for_data_w_valid ||
1027    s3_store_can_go_dup_for_data_w_valid ||
1028    s3_amo_can_go_dup_for_data_w_valid ||
1029    s3_miss_can_go_dup_for_data_w_valid ||
1030    s3_replace_can_go_dup_for_data_w_valid
1031  val s3_update_data_cango_dup_for_data_w_valid = s3_store_can_go_dup_for_data_w_valid || s3_amo_can_go_dup_for_data_w_valid || s3_miss_can_go_dup_for_data_w_valid
1032
1033  val s3_fire_dup_for_data_w_valid = s3_valid_dup_for_data_w_valid && s3_can_go_dup_for_data_w_valid
1034  when (do_amoalu_dup_for_data_w_valid) { s3_s_amoalu_dup_for_data_w_valid := true.B }
1035  when (s3_fire_dup_for_data_w_valid) { s3_s_amoalu_dup_for_data_w_valid := false.B }
1036
1037  val s3_banked_store_wmask_dup_for_data_w_valid = RegEnable(s2_banked_store_wmask, s2_fire_to_s3)
1038  val s3_req_word_idx_dup_for_data_w_valid = RegEnable(s2_req.word_idx, s2_fire_to_s3)
1039  val banked_wmask = Mux(
1040    s3_req_miss_dup_for_data_w_valid,
1041    banked_full_wmask,
1042    Mux(
1043      s3_store_hit_dup_for_data_w_valid,
1044      s3_banked_store_wmask_dup_for_data_w_valid,
1045      Mux(
1046        s3_can_do_amo_write_dup_for_data_w_valid,
1047        UIntToOH(s3_req_word_idx_dup_for_data_w_valid),
1048        banked_none_wmask
1049      )
1050    )
1051  )
1052  assert(!(s3_valid && banked_wmask.orR && !update_data))
1053
1054  val s3_sc_data_merged_dup_for_data_w_valid = Wire(Vec(DCacheBanks, UInt(DCacheSRAMRowBits.W)))
1055  val s3_req_amo_data_dup_for_data_w_valid = RegEnable(s2_req.amo_data, s2_fire_to_s3)
1056  val s3_req_amo_mask_dup_for_data_w_valid = RegEnable(s2_req.amo_mask, s2_fire_to_s3)
1057  for (i <- 0 until DCacheBanks) {
1058    val old_data = s3_store_data_merged(i)
1059    s3_sc_data_merged_dup_for_data_w_valid(i) := mergePutData(old_data, s3_req_amo_data_dup_for_data_w_valid,
1060      Mux(
1061        s3_req_word_idx_dup_for_data_w_valid === i.U && !s3_sc_fail_dup_for_data_w_valid,
1062        s3_req_amo_mask_dup_for_data_w_valid,
1063        0.U(wordBytes.W)
1064      )
1065    )
1066  }
1067
1068  when (s2_fire_to_s3) { s3_valid_dup_for_data_w_valid := true.B }
1069  .elsewhen (s3_fire_dup_for_data_w_valid) { s3_valid_dup_for_data_w_valid := false.B }
1070
1071  val s3_valid_dup_for_data_w_bank = RegInit(VecInit(Seq.fill(DCacheBanks)(false.B))) // TODO
1072  val data_write_ready_dup_for_data_w_bank = io.data_write_ready_dup.drop(dataWritePort).take(DCacheBanks)
1073  val tag_write_ready_dup_for_data_w_bank = io.tag_write_ready_dup.drop(dataWritePort).take(DCacheBanks)
1074  val wb_ready_dup_for_data_w_bank = io.wb_ready_dup.drop(dataWritePort).take(DCacheBanks)
1075  for (i <- 0 until DCacheBanks) {
1076    val s3_req_miss_dup_for_data_w_bank = RegEnable(s2_req.miss, s2_fire_to_s3)
1077    val s3_req_probe_dup_for_data_w_bank = RegEnable(s2_req.probe, s2_fire_to_s3)
1078    val s3_tag_match_dup_for_data_w_bank = RegEnable(s2_tag_match, s2_fire_to_s3)
1079    val s3_coh_dup_for_data_w_bank = RegEnable(s2_coh, s2_fire_to_s3)
1080    val s3_req_probe_param_dup_for_data_w_bank = RegEnable(s2_req.probe_param, s2_fire_to_s3)
1081    val (_, _, probe_new_coh_dup_for_data_w_bank) = s3_coh_dup_for_data_w_bank.onProbe(s3_req_probe_param_dup_for_data_w_bank)
1082    val s3_req_source_dup_for_data_w_bank = RegEnable(s2_req.source, s2_fire_to_s3)
1083    val s3_req_cmd_dup_for_data_w_bank = RegEnable(s2_req.cmd, s2_fire_to_s3)
1084    val s3_req_replace_dup_for_data_w_bank = RegEnable(s2_req.replace, s2_fire_to_s3)
1085    val s3_hit_coh_dup_for_data_w_bank = RegEnable(s2_hit_coh, s2_fire_to_s3)
1086    val s3_new_hit_coh_dup_for_data_w_bank = RegEnable(s2_new_hit_coh, s2_fire_to_s3)
1087
1088    val miss_update_meta_dup_for_data_w_bank = s3_req_miss_dup_for_data_w_bank
1089    val probe_update_meta_dup_for_data_w_bank = s3_req_probe_dup_for_data_w_bank && s3_tag_match_dup_for_data_w_bank && s3_coh_dup_for_data_w_bank =/= probe_new_coh_dup_for_data_w_bank
1090    val store_update_meta_dup_for_data_w_bank = s3_req_source_dup_for_data_w_bank === STORE_SOURCE.U &&
1091      !s3_req_probe_dup_for_data_w_bank &&
1092      s3_hit_coh_dup_for_data_w_bank =/= s3_new_hit_coh_dup_for_data_w_bank
1093    val amo_update_meta_dup_for_data_w_bank = s3_req_source_dup_for_data_w_bank === AMO_SOURCE.U &&
1094      !s3_req_probe_dup_for_data_w_bank &&
1095      s3_hit_coh_dup_for_data_w_bank =/= s3_new_hit_coh_dup_for_data_w_bank
1096    val update_meta_dup_for_data_w_bank = (
1097      miss_update_meta_dup_for_data_w_bank ||
1098      probe_update_meta_dup_for_data_w_bank ||
1099      store_update_meta_dup_for_data_w_bank ||
1100      amo_update_meta_dup_for_data_w_bank
1101    ) && !s3_req_replace_dup_for_data_w_bank
1102
1103    val s3_amo_hit_dup_for_data_w_bank = RegEnable(s2_amo_hit, s2_fire_to_s3)
1104    val s3_s_amoalu_dup_for_data_w_bank = RegInit(false.B)
1105    val amo_wait_amoalu_dup_for_data_w_bank = s3_req_source_dup_for_data_w_bank === AMO_SOURCE.U &&
1106      s3_req_cmd_dup_for_data_w_bank =/= M_XLR &&
1107      s3_req_cmd_dup_for_data_w_bank =/= M_XSC
1108    val do_amoalu_dup_for_data_w_bank = amo_wait_amoalu_dup_for_data_w_bank && s3_valid_dup_for_data_w_bank(i) && !s3_s_amoalu_dup_for_data_w_bank
1109
1110    val s3_store_hit_dup_for_data_w_bank = RegEnable(s2_store_hit, s2_fire_to_s3)
1111    val s3_req_addr_dup_for_data_w_bank = RegEnable(s2_req.addr, s2_fire_to_s3)
1112    val s3_can_do_amo_dup_for_data_w_bank = (s3_req_miss_dup_for_data_w_bank && !s3_req_probe_dup_for_data_w_bank && s3_req_source_dup_for_data_w_bank === AMO_SOURCE.U) ||
1113      s3_amo_hit_dup_for_data_w_bank
1114
1115    val s3_lr_dup_for_data_w_bank = !s3_req_probe_dup_for_data_w_bank && s3_req_source_dup_for_data_w_bank === AMO_SOURCE.U && s3_req_cmd_dup_for_data_w_bank === M_XLR
1116    val s3_sc_dup_for_data_w_bank = !s3_req_probe_dup_for_data_w_bank && s3_req_source_dup_for_data_w_bank === AMO_SOURCE.U && s3_req_cmd_dup_for_data_w_bank === M_XSC
1117    val lrsc_addr_dup_for_data_w_bank = Reg(UInt())
1118    val lrsc_count_dup_for_data_w_bank = RegInit(0.U(log2Ceil(LRSCCycles).W))
1119
1120    when (s3_valid_dup_for_data_w_bank(i) && (s3_lr_dup_for_data_w_bank || s3_sc_dup_for_data_w_bank)) {
1121      when (s3_can_do_amo_dup_for_data_w_bank && s3_lr_dup_for_data_w_bank) {
1122        lrsc_count_dup_for_data_w_bank := (LRSCCycles - 1).U
1123        lrsc_addr_dup_for_data_w_bank := get_block_addr(s3_req_addr_dup_for_data_w_bank)
1124      }.otherwise {
1125        lrsc_count_dup_for_data_w_bank := 0.U
1126      }
1127    }.elsewhen (io.invalid_resv_set) {
1128      lrsc_count_dup_for_data_w_bank := 0.U
1129    }.elsewhen (lrsc_count_dup_for_data_w_bank > 0.U) {
1130      lrsc_count_dup_for_data_w_bank := lrsc_count_dup_for_data_w_bank - 1.U
1131    }
1132
1133    val lrsc_valid_dup_for_data_w_bank = lrsc_count_dup_for_data_w_bank > LRSCBackOff.U
1134    val s3_lrsc_addr_match_dup_for_data_w_bank = lrsc_valid_dup_for_data_w_bank && lrsc_addr_dup_for_data_w_bank === get_block_addr(s3_req_addr_dup_for_data_w_bank)
1135    val s3_sc_fail_dup_for_data_w_bank = s3_sc_dup_for_data_w_bank && !s3_lrsc_addr_match_dup_for_data_w_bank
1136    val s3_can_do_amo_write_dup_for_data_w_bank = s3_can_do_amo_dup_for_data_w_bank && isWrite(s3_req_cmd_dup_for_data_w_bank) && !s3_sc_fail_dup_for_data_w_bank
1137    val update_data_dup_for_data_w_bank = s3_req_miss_dup_for_data_w_bank || s3_store_hit_dup_for_data_w_bank || s3_can_do_amo_write_dup_for_data_w_bank
1138
1139    val s3_probe_can_go_dup_for_data_w_bank = s3_req_probe_dup_for_data_w_bank &&
1140      wb_ready_dup_for_data_w_bank(i) &&
1141      (io.meta_write.ready || !probe_update_meta_dup_for_data_w_bank)
1142    val s3_store_can_go_dup_for_data_w_bank = s3_req_source_dup_for_data_w_bank === STORE_SOURCE.U && !s3_req_probe_dup_for_data_w_bank &&
1143      (io.meta_write.ready || !store_update_meta_dup_for_data_w_bank) &&
1144      (data_write_ready_dup_for_data_w_bank(i) || !update_data_dup_for_data_w_bank)
1145    val s3_amo_can_go_dup_for_data_w_bank = s3_amo_hit_dup_for_data_w_bank &&
1146      (io.meta_write.ready || !amo_update_meta_dup_for_data_w_bank) &&
1147      (data_write_ready_dup_for_data_w_bank(i) || !update_data_dup_for_data_w_bank) &&
1148      (s3_s_amoalu_dup_for_data_w_bank || !amo_wait_amoalu_dup_for_data_w_bank)
1149    val s3_miss_can_go_dup_for_data_w_bank = s3_req_miss_dup_for_data_w_bank &&
1150      (io.meta_write.ready || !amo_update_meta_dup_for_data_w_bank) &&
1151      (data_write_ready_dup_for_data_w_bank(i) || !update_data_dup_for_data_w_bank) &&
1152      (s3_s_amoalu_dup_for_data_w_bank || !amo_wait_amoalu_dup_for_data_w_bank) &&
1153      tag_write_ready_dup_for_data_w_bank(i) &&
1154      wb_ready_dup_for_data_w_bank(i)
1155    val s3_replace_can_go_dup_for_data_w_bank = s3_req_replace_dup_for_data_w_bank &&
1156      (s3_coh_dup_for_data_w_bank.state === ClientStates.Nothing || wb_ready_dup_for_data_w_bank(i))
1157    val s3_can_go_dup_for_data_w_bank = s3_probe_can_go_dup_for_data_w_bank ||
1158      s3_store_can_go_dup_for_data_w_bank ||
1159      s3_amo_can_go_dup_for_data_w_bank ||
1160      s3_miss_can_go_dup_for_data_w_bank ||
1161      s3_replace_can_go_dup_for_data_w_bank
1162    val s3_update_data_cango_dup_for_data_w_bank = s3_store_can_go_dup_for_data_w_bank || s3_amo_can_go_dup_for_data_w_bank || s3_miss_can_go_dup_for_data_w_bank
1163
1164    val s3_fire_dup_for_data_w_bank = s3_valid_dup_for_data_w_bank(i) && s3_can_go_dup_for_data_w_bank
1165
1166    when (do_amoalu_dup_for_data_w_bank) { s3_s_amoalu_dup_for_data_w_bank := true.B }
1167    when (s3_fire_dup_for_data_w_bank) { s3_s_amoalu_dup_for_data_w_bank := false.B }
1168
1169    when (s2_fire_to_s3) { s3_valid_dup_for_data_w_bank(i) := true.B }
1170    .elsewhen (s3_fire_dup_for_data_w_bank) { s3_valid_dup_for_data_w_bank(i) := false.B }
1171
1172    io.data_write_dup(i).valid := s3_valid_dup_for_data_w_bank(i) && s3_update_data_cango_dup_for_data_w_bank && update_data_dup_for_data_w_bank
1173    io.data_write_dup(i).bits.way_en := RegEnable(s2_way_en, s2_fire_to_s3)
1174    io.data_write_dup(i).bits.addr := RegEnable(s2_req.vaddr, s2_fire_to_s3)
1175  }
1176  // -------------------------------------------------------------------------------------
1177
1178  // ---------------- duplicate regs for wb.valid to solve fanout ----------------
1179  val s3_req_miss_dup_for_wb_valid = RegEnable(s2_req.miss, s2_fire_to_s3)
1180  val s3_req_probe_dup_for_wb_valid = RegEnable(s2_req.probe, s2_fire_to_s3)
1181  val s3_tag_match_dup_for_wb_valid = RegEnable(s2_tag_match, s2_fire_to_s3)
1182  val s3_coh_dup_for_wb_valid = RegEnable(s2_coh, s2_fire_to_s3)
1183  val s3_req_probe_param_dup_for_wb_valid = RegEnable(s2_req.probe_param, s2_fire_to_s3)
1184  val (_, _, probe_new_coh_dup_for_wb_valid) = s3_coh_dup_for_wb_valid.onProbe(s3_req_probe_param_dup_for_wb_valid)
1185  val s3_req_source_dup_for_wb_valid = RegEnable(s2_req.source, s2_fire_to_s3)
1186  val s3_req_cmd_dup_for_wb_valid = RegEnable(s2_req.cmd, s2_fire_to_s3)
1187  val s3_req_replace_dup_for_wb_valid = RegEnable(s2_req.replace, s2_fire_to_s3)
1188  val s3_hit_coh_dup_for_wb_valid = RegEnable(s2_hit_coh, s2_fire_to_s3)
1189  val s3_new_hit_coh_dup_for_wb_valid = RegEnable(s2_new_hit_coh, s2_fire_to_s3)
1190
1191  val miss_update_meta_dup_for_wb_valid = s3_req_miss_dup_for_wb_valid
1192  val probe_update_meta_dup_for_wb_valid = s3_req_probe_dup_for_wb_valid && s3_tag_match_dup_for_wb_valid && s3_coh_dup_for_wb_valid =/= probe_new_coh_dup_for_wb_valid
1193  val store_update_meta_dup_for_wb_valid = s3_req_source_dup_for_wb_valid === STORE_SOURCE.U &&
1194    !s3_req_probe_dup_for_wb_valid &&
1195    s3_hit_coh_dup_for_wb_valid =/= s3_new_hit_coh_dup_for_wb_valid
1196  val amo_update_meta_dup_for_wb_valid = s3_req_source_dup_for_wb_valid === AMO_SOURCE.U &&
1197    !s3_req_probe_dup_for_wb_valid &&
1198    s3_hit_coh_dup_for_wb_valid =/= s3_new_hit_coh_dup_for_wb_valid
1199  val update_meta_dup_for_wb_valid = (
1200    miss_update_meta_dup_for_wb_valid ||
1201    probe_update_meta_dup_for_wb_valid ||
1202    store_update_meta_dup_for_wb_valid ||
1203    amo_update_meta_dup_for_wb_valid
1204  ) && !s3_req_replace_dup_for_wb_valid
1205
1206  val s3_valid_dup_for_wb_valid = RegInit(false.B)
1207  val s3_amo_hit_dup_for_wb_valid = RegEnable(s2_amo_hit, s2_fire_to_s3)
1208  val s3_s_amoalu_dup_for_wb_valid = RegInit(false.B)
1209  val amo_wait_amoalu_dup_for_wb_valid = s3_req_source_dup_for_wb_valid === AMO_SOURCE.U &&
1210    s3_req_cmd_dup_for_wb_valid =/= M_XLR &&
1211    s3_req_cmd_dup_for_wb_valid =/= M_XSC
1212  val do_amoalu_dup_for_wb_valid = amo_wait_amoalu_dup_for_wb_valid && s3_valid_dup_for_wb_valid && !s3_s_amoalu_dup_for_wb_valid
1213
1214  val s3_store_hit_dup_for_wb_valid = RegEnable(s2_store_hit, s2_fire_to_s3)
1215  val s3_req_addr_dup_for_wb_valid = RegEnable(s2_req.addr, s2_fire_to_s3)
1216  val s3_can_do_amo_dup_for_wb_valid = (s3_req_miss_dup_for_wb_valid && !s3_req_probe_dup_for_wb_valid && s3_req_source_dup_for_wb_valid === AMO_SOURCE.U) ||
1217    s3_amo_hit_dup_for_wb_valid
1218
1219  val s3_lr_dup_for_wb_valid = !s3_req_probe_dup_for_wb_valid && s3_req_source_dup_for_wb_valid === AMO_SOURCE.U && s3_req_cmd_dup_for_wb_valid === M_XLR
1220  val s3_sc_dup_for_wb_valid = !s3_req_probe_dup_for_wb_valid && s3_req_source_dup_for_wb_valid === AMO_SOURCE.U && s3_req_cmd_dup_for_wb_valid === M_XSC
1221  val lrsc_addr_dup_for_wb_valid = Reg(UInt())
1222  val lrsc_count_dup_for_wb_valid = RegInit(0.U(log2Ceil(LRSCCycles).W))
1223
1224  when (s3_valid_dup_for_wb_valid && (s3_lr_dup_for_wb_valid || s3_sc_dup_for_wb_valid)) {
1225    when (s3_can_do_amo_dup_for_wb_valid && s3_lr_dup_for_wb_valid) {
1226      lrsc_count_dup_for_wb_valid := (LRSCCycles - 1).U
1227      lrsc_addr_dup_for_wb_valid := get_block_addr(s3_req_addr_dup_for_wb_valid)
1228    }.otherwise {
1229      lrsc_count_dup_for_wb_valid := 0.U
1230    }
1231  }.elsewhen (io.invalid_resv_set) {
1232    lrsc_count_dup_for_wb_valid := 0.U
1233  }.elsewhen (lrsc_count_dup_for_wb_valid > 0.U) {
1234    lrsc_count_dup_for_wb_valid := lrsc_count_dup_for_wb_valid - 1.U
1235  }
1236
1237  val lrsc_valid_dup_for_wb_valid = lrsc_count_dup_for_wb_valid > LRSCBackOff.U
1238  val s3_lrsc_addr_match_dup_for_wb_valid = lrsc_valid_dup_for_wb_valid && lrsc_addr_dup_for_wb_valid === get_block_addr(s3_req_addr_dup_for_wb_valid)
1239  val s3_sc_fail_dup_for_wb_valid = s3_sc_dup_for_wb_valid && !s3_lrsc_addr_match_dup_for_wb_valid
1240  val s3_can_do_amo_write_dup_for_wb_valid = s3_can_do_amo_dup_for_wb_valid && isWrite(s3_req_cmd_dup_for_wb_valid) && !s3_sc_fail_dup_for_wb_valid
1241  val update_data_dup_for_wb_valid = s3_req_miss_dup_for_wb_valid || s3_store_hit_dup_for_wb_valid || s3_can_do_amo_write_dup_for_wb_valid
1242
1243  val s3_probe_can_go_dup_for_wb_valid = s3_req_probe_dup_for_wb_valid &&
1244    io.wb_ready_dup(wbPort) &&
1245    (io.meta_write.ready || !probe_update_meta_dup_for_wb_valid)
1246  val s3_store_can_go_dup_for_wb_valid = s3_req_source_dup_for_wb_valid === STORE_SOURCE.U && !s3_req_probe_dup_for_wb_valid &&
1247    (io.meta_write.ready || !store_update_meta_dup_for_wb_valid) &&
1248    (io.data_write_ready_dup(wbPort) || !update_data_dup_for_wb_valid)
1249  val s3_amo_can_go_dup_for_wb_valid = s3_amo_hit_dup_for_wb_valid &&
1250    (io.meta_write.ready || !amo_update_meta_dup_for_wb_valid) &&
1251    (io.data_write_ready_dup(wbPort) || !update_data_dup_for_wb_valid) &&
1252    (s3_s_amoalu_dup_for_wb_valid || !amo_wait_amoalu_dup_for_wb_valid)
1253  val s3_miss_can_go_dup_for_wb_valid = s3_req_miss_dup_for_wb_valid &&
1254    (io.meta_write.ready || !amo_update_meta_dup_for_wb_valid) &&
1255    (io.data_write_ready_dup(wbPort) || !update_data_dup_for_wb_valid) &&
1256    (s3_s_amoalu_dup_for_wb_valid || !amo_wait_amoalu_dup_for_wb_valid) &&
1257    io.tag_write_ready_dup(wbPort) &&
1258    io.wb_ready_dup(wbPort)
1259  val s3_replace_can_go_dup_for_wb_valid = s3_req_replace_dup_for_wb_valid &&
1260    (s3_coh_dup_for_wb_valid.state === ClientStates.Nothing || io.wb_ready_dup(wbPort))
1261  val s3_can_go_dup_for_wb_valid = s3_probe_can_go_dup_for_wb_valid ||
1262    s3_store_can_go_dup_for_wb_valid ||
1263    s3_amo_can_go_dup_for_wb_valid ||
1264    s3_miss_can_go_dup_for_wb_valid ||
1265    s3_replace_can_go_dup_for_wb_valid
1266  val s3_update_data_cango_dup_for_wb_valid = s3_store_can_go_dup_for_wb_valid || s3_amo_can_go_dup_for_wb_valid || s3_miss_can_go_dup_for_wb_valid
1267
1268  val s3_fire_dup_for_wb_valid = s3_valid_dup_for_wb_valid && s3_can_go_dup_for_wb_valid
1269  when (do_amoalu_dup_for_wb_valid) { s3_s_amoalu_dup_for_wb_valid := true.B }
1270  when (s3_fire_dup_for_wb_valid) { s3_s_amoalu_dup_for_wb_valid := false.B }
1271
1272  val s3_banked_store_wmask_dup_for_wb_valid = RegEnable(s2_banked_store_wmask, s2_fire_to_s3)
1273  val s3_req_word_idx_dup_for_wb_valid = RegEnable(s2_req.word_idx, s2_fire_to_s3)
1274  val s3_replace_nothing_dup_for_wb_valid = s3_req_replace_dup_for_wb_valid && s3_coh_dup_for_wb_valid.state === ClientStates.Nothing
1275
1276  val s3_sc_data_merged_dup_for_wb_valid = Wire(Vec(DCacheBanks, UInt(DCacheSRAMRowBits.W)))
1277  val s3_req_amo_data_dup_for_wb_valid = RegEnable(s2_req.amo_data, s2_fire_to_s3)
1278  val s3_req_amo_mask_dup_for_wb_valid = RegEnable(s2_req.amo_mask, s2_fire_to_s3)
1279  for (i <- 0 until DCacheBanks) {
1280    val old_data = s3_store_data_merged(i)
1281    s3_sc_data_merged_dup_for_wb_valid(i) := mergePutData(old_data, s3_req_amo_data_dup_for_wb_valid,
1282      Mux(
1283        s3_req_word_idx_dup_for_wb_valid === i.U && !s3_sc_fail_dup_for_wb_valid,
1284        s3_req_amo_mask_dup_for_wb_valid,
1285        0.U(wordBytes.W)
1286      )
1287    )
1288  }
1289
1290  val s3_need_replacement_dup_for_wb_valid = RegEnable(s2_need_replacement, s2_fire_to_s3)
1291  val miss_wb_dup_for_wb_valid = s3_req_miss_dup_for_wb_valid && s3_need_replacement_dup_for_wb_valid &&
1292    s3_coh_dup_for_wb_valid.state =/= ClientStates.Nothing
1293  val need_wb_dup_for_wb_valid = miss_wb_dup_for_wb_valid || s3_req_probe_dup_for_wb_valid || s3_req_replace_dup_for_wb_valid
1294
1295  val s3_tag_dup_for_wb_valid = RegEnable(s2_tag, s2_fire_to_s3)
1296
1297  val (_, probe_shrink_param_dup_for_wb_valid, _) = s3_coh_dup_for_wb_valid.onProbe(s3_req_probe_param_dup_for_wb_valid)
1298  val (_, miss_shrink_param_dup_for_wb_valid, _) = s3_coh_dup_for_wb_valid.onCacheControl(M_FLUSH)
1299  val writeback_param_dup_for_wb_valid = Mux(
1300    s3_req_probe_dup_for_wb_valid,
1301    probe_shrink_param_dup_for_wb_valid,
1302    miss_shrink_param_dup_for_wb_valid
1303  )
1304  val writeback_data_dup_for_wb_valid = if (dcacheParameters.alwaysReleaseData) {
1305    s3_tag_match_dup_for_wb_valid && s3_req_probe_dup_for_wb_valid && RegEnable(s2_req.probe_need_data, s2_fire_to_s3) ||
1306      s3_coh_dup_for_wb_valid === ClientStates.Dirty || (miss_wb_dup_for_wb_valid || s3_req_replace_dup_for_wb_valid) && s3_coh_dup_for_wb_valid.state =/= ClientStates.Nothing
1307  } else {
1308    s3_tag_match_dup_for_wb_valid && s3_req_probe_dup_for_wb_valid && RegEnable(s2_req.probe_need_data, s2_fire_to_s3) || s3_coh_dup_for_wb_valid === ClientStates.Dirty
1309  }
1310
1311  when (s2_fire_to_s3) { s3_valid_dup_for_wb_valid := true.B }
1312  .elsewhen (s3_fire_dup_for_wb_valid) { s3_valid_dup_for_wb_valid := false.B }
1313
1314  // -------------------------------------------------------------------------------------
1315
1316  val s3_fire = s3_valid_dup(4) && s3_can_go
1317  when (s2_fire_to_s3) {
1318    s3_valid := true.B
1319    s3_valid_dup.foreach(_ := true.B)
1320    s3_valid_dup_for_status.foreach(_ := true.B)
1321  }.elsewhen (s3_fire) {
1322    s3_valid := false.B
1323    s3_valid_dup.foreach(_ := false.B)
1324    s3_valid_dup_for_status.foreach(_ := false.B)
1325  }
1326  s3_ready := !s3_valid_dup(5) || s3_can_go
1327  s3_s0_set_conflict := s3_valid_dup(6) && s3_idx_dup(0) === s0_idx
1328  s3_s0_set_conflict_store := s3_valid_dup(7) && s3_idx_dup(1) === store_idx
1329  assert(RegNext(!s3_valid || !(s3_req_source_dup_2 === STORE_SOURCE.U && !s3_req.probe) || s3_hit)) // miss store should never come to s3
1330
1331  when(s3_fire) {
1332    s3_s_amoalu := false.B
1333    s3_s_amoalu_dup.foreach(_ := false.B)
1334  }
1335
1336  req.ready := s0_can_go
1337
1338  io.meta_read.valid := req.valid && s1_ready && !set_conflict
1339  io.meta_read.bits.idx := get_idx(s0_req.vaddr)
1340  io.meta_read.bits.way_en := Mux(s0_req.replace, s0_req.replace_way_en, ~0.U(nWays.W))
1341
1342  io.tag_read.valid := req.valid && s1_ready && !set_conflict && !s0_req.replace
1343  io.tag_read.bits.idx := get_idx(s0_req.vaddr)
1344  io.tag_read.bits.way_en := ~0.U(nWays.W)
1345
1346  io.data_read_intend := s1_valid_dup(3) && s1_need_data
1347  io.data_read.valid := s1_valid_dup(4) && s1_need_data && s2_ready
1348  io.data_read.bits.rmask := s1_banked_rmask
1349  io.data_read.bits.way_en := s1_way_en
1350  io.data_read.bits.addr := s1_req_vaddr_dup_for_data_read
1351
1352  io.miss_req.valid := s2_valid_dup(4) && s2_can_go_to_mq_dup(0)
1353  val miss_req = io.miss_req.bits
1354  miss_req := DontCare
1355  miss_req.source := s2_req.source
1356  miss_req.cmd := s2_req.cmd
1357  miss_req.addr := s2_req.addr
1358  miss_req.vaddr := s2_req_vaddr_dup_for_miss_req
1359  miss_req.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_repl_way_en)
1360  miss_req.store_data := s2_req.store_data
1361  miss_req.store_mask := s2_req.store_mask
1362  miss_req.word_idx := s2_req.word_idx
1363  miss_req.amo_data := s2_req.amo_data
1364  miss_req.amo_mask := s2_req.amo_mask
1365  miss_req.req_coh := s2_hit_coh
1366  miss_req.replace_coh := s2_repl_coh
1367  miss_req.replace_tag := s2_repl_tag
1368  miss_req.id := s2_req.id
1369  miss_req.cancel := false.B
1370
1371  io.store_replay_resp.valid := s2_valid_dup(5) && s2_can_go_to_mq_dup(1) && replay && s2_req.isStore
1372  io.store_replay_resp.bits.data := DontCare
1373  io.store_replay_resp.bits.miss := true.B
1374  io.store_replay_resp.bits.replay := true.B
1375  io.store_replay_resp.bits.id := s2_req.id
1376
1377  io.store_hit_resp.valid := s3_valid_dup(8) && s3_store_can_go
1378  io.store_hit_resp.bits.data := DontCare
1379  io.store_hit_resp.bits.miss := false.B
1380  io.store_hit_resp.bits.replay := false.B
1381  io.store_hit_resp.bits.id := s3_req.id
1382
1383  io.release_update.valid := s3_valid_dup(9) && (s3_store_can_go || s3_amo_can_go) && s3_hit && update_data
1384  io.release_update.bits.addr := s3_req_addr_dup(3)
1385  io.release_update.bits.mask := Mux(s3_store_hit_dup(1), s3_banked_store_wmask, banked_amo_wmask)
1386  io.release_update.bits.data := Mux(
1387    amo_wait_amoalu,
1388    s3_amo_data_merged_reg,
1389    Mux(
1390      s3_sc,
1391      s3_sc_data_merged,
1392      s3_store_data_merged
1393    )
1394  ).asUInt
1395
1396  val atomic_hit_resp = Wire(new AtomicsResp)
1397  atomic_hit_resp.data := Mux(s3_sc, s3_sc_resp, s3_data_word)
1398  atomic_hit_resp.miss := false.B
1399  atomic_hit_resp.miss_id := s3_req.miss_id
1400  atomic_hit_resp.error := s3_error
1401  atomic_hit_resp.replay := false.B
1402  atomic_hit_resp.ack_miss_queue := s3_req_miss_dup(5)
1403  atomic_hit_resp.id := lrsc_valid_dup(2)
1404  val atomic_replay_resp = Wire(new AtomicsResp)
1405  atomic_replay_resp.data := DontCare
1406  atomic_replay_resp.miss := true.B
1407  atomic_replay_resp.miss_id := DontCare
1408  atomic_replay_resp.error := false.B
1409  atomic_replay_resp.replay := true.B
1410  atomic_replay_resp.ack_miss_queue := false.B
1411  atomic_replay_resp.id := DontCare
1412  val atomic_replay_resp_valid = s2_valid_dup(6) && s2_can_go_to_mq_dup(2) && replay && s2_req.isAMO
1413  val atomic_hit_resp_valid = s3_valid_dup(10) && (s3_amo_can_go || s3_miss_can_go && s3_req.isAMO)
1414  io.atomic_resp.valid := atomic_replay_resp_valid || atomic_hit_resp_valid
1415  io.atomic_resp.bits := Mux(atomic_replay_resp_valid, atomic_replay_resp, atomic_hit_resp)
1416
1417  io.replace_resp.valid := s3_fire && s3_req_replace_dup(3)
1418  io.replace_resp.bits := s3_req.miss_id
1419
1420  io.meta_write.valid := s3_fire_dup_for_meta_w_valid && update_meta_dup_for_meta_w_valid
1421  io.meta_write.bits.idx := s3_idx_dup(2)
1422  io.meta_write.bits.way_en := s3_way_en_dup(0)
1423  io.meta_write.bits.meta.coh := new_coh
1424
1425  io.error_flag_write.valid := s3_fire_dup_for_err_w_valid && update_meta_dup_for_err_w_valid && s3_l2_error
1426  io.error_flag_write.bits.idx := s3_idx_dup(3)
1427  io.error_flag_write.bits.way_en := s3_way_en_dup(1)
1428  io.error_flag_write.bits.error := s3_l2_error
1429
1430  io.tag_write.valid := s3_fire_dup_for_tag_w_valid && s3_req_miss_dup_for_tag_w_valid
1431  io.tag_write.bits.idx := s3_idx_dup(4)
1432  io.tag_write.bits.way_en := s3_way_en_dup(2)
1433  io.tag_write.bits.tag := get_tag(s3_req_addr_dup(4))
1434
1435  io.tag_write_intend := s3_req_miss_dup(7) && s3_valid_dup(11)
1436  XSPerfAccumulate("fake_tag_write_intend", io.tag_write_intend && !io.tag_write.valid)
1437  XSPerfAccumulate("mainpipe_tag_write", io.tag_write.valid)
1438
1439  assert(!RegNext(io.tag_write.valid && !io.tag_write_intend))
1440
1441  io.data_write.valid := s3_valid_dup_for_data_w_valid && s3_update_data_cango_dup_for_data_w_valid && update_data_dup_for_data_w_valid
1442  io.data_write.bits.way_en := s3_way_en_dup(3)
1443  io.data_write.bits.addr := s3_req_vaddr_dup_for_data_write
1444  io.data_write.bits.wmask := banked_wmask
1445  io.data_write.bits.data := Mux(
1446    amo_wait_amoalu_dup_for_data_w_valid,
1447    s3_amo_data_merged_reg,
1448    Mux(
1449      s3_sc_dup_for_data_w_valid,
1450      s3_sc_data_merged_dup_for_data_w_valid,
1451      s3_store_data_merged
1452    )
1453  )
1454  assert(RegNext(!io.meta_write.valid || !s3_req.replace))
1455  assert(RegNext(!io.tag_write.valid || !s3_req.replace))
1456  assert(RegNext(!io.data_write.valid || !s3_req.replace))
1457
1458  io.wb.valid := s3_valid_dup_for_wb_valid && (
1459    // replace
1460    s3_req_replace_dup_for_wb_valid && !s3_replace_nothing_dup_for_wb_valid ||
1461    // probe can go to wbq
1462    s3_req_probe_dup_for_wb_valid && (io.meta_write.ready || !probe_update_meta_dup_for_wb_valid) ||
1463      // amo miss can go to wbq
1464      s3_req_miss_dup_for_wb_valid &&
1465        (io.meta_write.ready || !amo_update_meta_dup_for_wb_valid) &&
1466        (io.data_write_ready_dup(wbPort) || !update_data_dup_for_wb_valid) &&
1467        (s3_s_amoalu_dup_for_wb_valid || !amo_wait_amoalu_dup_for_wb_valid) &&
1468        io.tag_write_ready_dup(wbPort)
1469    ) && need_wb_dup_for_wb_valid
1470
1471  io.wb.bits.addr := get_block_addr(Cat(s3_tag_dup_for_wb_valid, get_untag(s3_req.vaddr)))
1472  io.wb.bits.param := writeback_param_dup_for_wb_valid
1473  io.wb.bits.voluntary := s3_req_miss_dup_for_wb_valid || s3_req_replace_dup_for_wb_valid
1474  io.wb.bits.hasData := writeback_data_dup_for_wb_valid
1475  io.wb.bits.dirty := s3_coh_dup_for_wb_valid === ClientStates.Dirty
1476  io.wb.bits.data := s3_data.asUInt()
1477  io.wb.bits.delay_release := s3_req_replace_dup_for_wb_valid
1478  io.wb.bits.miss_id := s3_req.miss_id
1479
1480  io.replace_access.valid := RegNext(s1_fire && (s1_req.isAMO || s1_req.isStore) && !s1_req.probe)
1481  io.replace_access.bits.set := s2_idx_dup_for_replace_access
1482  io.replace_access.bits.way := RegNext(OHToUInt(s1_way_en))
1483
1484  io.replace_way.set.valid := RegNext(s0_fire)
1485  io.replace_way.set.bits := s1_idx_dup_for_replace_way
1486
1487  // TODO: consider block policy of a finer granularity
1488  io.status.s0_set.valid := req.valid
1489  io.status.s0_set.bits := get_idx(s0_req.vaddr)
1490  io.status.s1.valid := s1_valid_dup(5)
1491  io.status.s1.bits.set := s1_idx
1492  io.status.s1.bits.way_en := s1_way_en
1493  io.status.s2.valid := s2_valid_dup(7) && !s2_req_replace_dup_2
1494  io.status.s2.bits.set := s2_idx_dup_for_status
1495  io.status.s2.bits.way_en := s2_way_en
1496  io.status.s3.valid := s3_valid && !s3_req_replace_dup(7)
1497  io.status.s3.bits.set := s3_idx_dup(5)
1498  io.status.s3.bits.way_en := s3_way_en
1499
1500  for ((s, i) <- io.status_dup.zipWithIndex) {
1501    s.s1.valid := s1_valid_dup_for_status(i)
1502    s.s1.bits.set := RegEnable(get_idx(s0_req.vaddr), s0_fire)
1503    s.s1.bits.way_en := s1_way_en
1504    s.s2.valid := s2_valid_dup_for_status(i) && !RegEnable(s1_req.replace, s1_fire)
1505    s.s2.bits.set := RegEnable(get_idx(s1_req.vaddr), s1_fire)
1506    s.s2.bits.way_en := RegEnable(s1_way_en, s1_fire)
1507    s.s3.valid := s3_valid_dup_for_status(i) && !RegEnable(s2_req.replace, s2_fire_to_s3)
1508    s.s3.bits.set := RegEnable(get_idx(s2_req.vaddr), s2_fire_to_s3)
1509    s.s3.bits.way_en := RegEnable(s2_way_en, s2_fire_to_s3)
1510  }
1511  dontTouch(io.status_dup)
1512
1513  // report error to beu and csr, 1 cycle after read data resp
1514  io.error := 0.U.asTypeOf(new L1CacheErrorInfo())
1515  // report error, update error csr
1516  io.error.valid := s3_error && RegNext(s2_fire)
1517  // only tag_error and data_error will be reported to beu
1518  // l2_error should not be reported (l2 will report that)
1519  io.error.report_to_beu := (RegEnable(s2_tag_error, s2_fire) || s3_data_error) && RegNext(s2_fire)
1520  io.error.paddr := RegEnable(s2_req.addr, s2_fire)
1521  io.error.source.tag := RegEnable(s2_tag_error, s2_fire)
1522  io.error.source.data := s3_data_error
1523  io.error.source.l2 := RegEnable(s2_flag_error || s2_l2_error, s2_fire)
1524  io.error.opType.store := RegEnable(s2_req.isStore && !s2_req.probe, s2_fire)
1525  io.error.opType.probe := RegEnable(s2_req.probe, s2_fire)
1526  io.error.opType.release := RegEnable(s2_req.replace, s2_fire)
1527  io.error.opType.atom := RegEnable(s2_req.isAMO && !s2_req.probe, s2_fire)
1528
1529  val perfEvents = Seq(
1530    ("dcache_mp_req          ", s0_fire                                                      ),
1531    ("dcache_mp_total_penalty", PopCount(VecInit(Seq(s0_fire, s1_valid, s2_valid, s3_valid))))
1532  )
1533  generatePerfEvent()
1534}
1535