1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan._ 25import xiangshan.ExceptionNO._ 26import xiangshan.backend.Bundles.{DynInst, MemExuInput, MemExuOutput} 27import xiangshan.backend.fu.PMPRespBundle 28import xiangshan.backend.fu.FuConfig._ 29import xiangshan.backend.ctrlblock.{DebugLsInfoBundle, LsTopdownInfo} 30import xiangshan.backend.fu.NewCSR._ 31import xiangshan.backend.rob.RobPtr 32import xiangshan.backend.fu._ 33import xiangshan.backend.fu.util.SdtrigExt 34import xiangshan.mem.mdp._ 35import xiangshan.mem.Bundles._ 36import xiangshan.cache._ 37import xiangshan.cache.wpu.ReplayCarry 38import xiangshan.cache.mmu.{TlbCmd, TlbHintReq, TlbReq, TlbRequestIO, TlbResp} 39 40class HybridUnit(implicit p: Parameters) extends XSModule 41 with HasLoadHelper 42 with HasPerfEvents 43 with HasDCacheParameters 44 with HasCircularQueuePtrHelper 45 with HasVLSUParameters 46 with SdtrigExt 47{ 48 val io = IO(new Bundle() { 49 // control 50 val redirect = Flipped(ValidIO(new Redirect)) 51 val csrCtrl = Flipped(new CustomCSRCtrlIO) 52 53 // flow in 54 val lsin = Flipped(Decoupled(new MemExuInput)) 55 56 // flow out 57 val ldout = DecoupledIO(new MemExuOutput) 58 val stout = DecoupledIO(new MemExuOutput) 59 60 val ldu_io = new Bundle() { 61 // dcache 62 val dcache = new DCacheLoadIO 63 64 // data path 65 val sbuffer = new LoadForwardQueryIO 66 val ubuffer = new LoadForwardQueryIO 67 val vec_forward = new LoadForwardQueryIO 68 val lsq = new LoadToLsqIO 69 val tl_d_channel = Input(new DcacheToLduForwardIO) 70 val forward_mshr = Flipped(new LduToMissqueueForwardIO) 71 val tlb_hint = Flipped(new TlbHintReq) 72 val l2_hint = Input(Valid(new L2ToL1Hint)) 73 74 // fast wakeup 75 val fast_uop = ValidIO(new DynInst) // early wakeup signal generated in load_s1, send to RS in load_s2 76 77 // trigger 78 val trigger = Vec(TriggerNum, new LoadUnitTriggerIO) 79 80 // load to load fast path 81 val l2l_fwd_in = Input(new LoadToLoadIO) 82 val l2l_fwd_out = Output(new LoadToLoadIO) 83 84 val ld_fast_match = Input(Bool()) 85 val ld_fast_fuOpType = Input(UInt()) 86 val ld_fast_imm = Input(UInt(12.W)) 87 88 // hardware prefetch to l1 cache req 89 val prefetch_req = Flipped(ValidIO(new L1PrefetchReq)) 90 91 // iq cancel 92 val ldCancel = Output(new LoadCancelIO()) // use to cancel the uops waked by this load, and cancel load 93 94 // iq wakeup, use to wakeup consumer uop at load s2 95 val wakeup = ValidIO(new DynInst) 96 97 // load ecc error 98 val s3_dly_ld_err = Output(Bool()) // Note that io.s3_dly_ld_err and io.lsq.s3_dly_ld_err is different 99 100 // schedule error query 101 val stld_nuke_query = Flipped(Vec(StorePipelineWidth, Valid(new StoreNukeQueryIO))) 102 103 // queue-based replay 104 val replay = Flipped(Decoupled(new LsPipelineBundle)) 105 val lq_rep_full = Input(Bool()) 106 107 // misc 108 val s2_ptr_chasing = Output(Bool()) // provide right pc for hw prefetch 109 110 // Load fast replay path 111 val fast_rep_in = Flipped(Decoupled(new LqWriteBundle)) 112 val fast_rep_out = Decoupled(new LqWriteBundle) 113 114 // Load RAR rollback 115 val rollback = Valid(new Redirect) 116 117 // perf 118 val debug_ls = Output(new DebugLsInfoBundle) 119 val lsTopdownInfo = Output(new LsTopdownInfo) 120 } 121 122 val stu_io = new Bundle() { 123 val dcache = new DCacheStoreIO 124 val prefetch_req = Flipped(DecoupledIO(new StorePrefetchReq)) 125 val issue = Valid(new MemExuInput) 126 val lsq = ValidIO(new LsPipelineBundle) 127 val lsq_replenish = Output(new LsPipelineBundle()) 128 val stld_nuke_query = Valid(new StoreNukeQueryIO) 129 val st_mask_out = Valid(new StoreMaskBundle) 130 val debug_ls = Output(new DebugLsInfoBundle) 131 } 132 133 val vec_stu_io = new Bundle() { 134 val in = Flipped(DecoupledIO(new VecPipeBundle())) 135 val isFirstIssue = Input(Bool()) 136 val lsq = ValidIO(new LsPipelineBundle()) 137 val feedbackSlow = ValidIO(new VSFQFeedback) 138 } 139 140 // speculative for gated control 141 val s0_prefetch_spec = Output(Bool()) 142 val s1_prefetch_spec = Output(Bool()) 143 // prefetch 144 val prefetch_train = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to sms 145 val prefetch_train_l1 = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to stream & stride 146 val canAcceptLowConfPrefetch = Output(Bool()) 147 val canAcceptHighConfPrefetch = Output(Bool()) 148 val correctMissTrain = Input(Bool()) 149 150 // data path 151 val tlb = new TlbRequestIO(2) 152 val pmp = Flipped(new PMPRespBundle()) // arrive same to tlb now 153 154 // rs feedback 155 val feedback_fast = ValidIO(new RSFeedback) // stage 2 156 val feedback_slow = ValidIO(new RSFeedback) // stage 3 157 158 // for store trigger 159 val fromCsrTrigger = Input(new CsrTriggerBundle) 160 }) 161 162 val StorePrefetchL1Enabled = EnableStorePrefetchAtCommit || EnableStorePrefetchAtIssue || EnableStorePrefetchSPB 163 val s1_ready, s2_ready, s3_ready, sx_can_go = WireInit(false.B) 164 165 // Pipeline 166 // -------------------------------------------------------------------------------- 167 // stage 0 168 // -------------------------------------------------------------------------------- 169 // generate addr, use addr to query DCache and DTLB 170 val s0_valid = Wire(Bool()) 171 val s0_dcache_ready = Wire(Bool()) 172 val s0_kill = Wire(Bool()) 173 val s0_vaddr = Wire(UInt(VAddrBits.W)) 174 val s0_mask = Wire(UInt((VLEN/8).W)) 175 val s0_uop = Wire(new DynInst) 176 val s0_has_rob_entry = Wire(Bool()) 177 val s0_mshrid = Wire(UInt()) 178 val s0_try_l2l = Wire(Bool()) 179 val s0_rep_carry = Wire(new ReplayCarry(nWays)) 180 val s0_isFirstIssue = Wire(Bool()) 181 val s0_fast_rep = Wire(Bool()) 182 val s0_ld_rep = Wire(Bool()) 183 val s0_l2l_fwd = Wire(Bool()) 184 val s0_sched_idx = Wire(UInt()) 185 val s0_can_go = s1_ready 186 val s0_fire = s0_valid && s0_dcache_ready && s0_can_go 187 val s0_out = Wire(new LqWriteBundle) 188 // vector 189 val s0_isvec = WireInit(false.B) 190 val s0_vecActive = WireInit(true.B) 191 // val s0_flowPtr = WireInit(0.U.asTypeOf(new VsFlowPtr)) 192 val s0_isLastElem = WireInit(false.B) 193 194 // load flow select/gen 195 // src0: super load replayed by LSQ (cache miss replay) (io.ldu_io.replay) 196 // src1: fast load replay (io.ldu_io.fast_rep_in) 197 // src2: load replayed by LSQ (io.ldu_io.replay) 198 // src3: hardware prefetch from prefetchor (high confidence) (io.prefetch) 199 // src4: int read / software prefetch first issue from RS (io.in) 200 // src5: vec read first issue from RS (TODO) 201 // src6: load try pointchaising when no issued or replayed load (io.fastpath) 202 // src7: hardware prefetch from prefetchor (high confidence) (io.prefetch) 203 // priority: high to low 204 val s0_ld_flow = FuType.isLoad(s0_uop.fuType) || FuType.isVLoad(s0_uop.fuType) 205 val s0_rep_stall = io.lsin.valid && isAfter(io.ldu_io.replay.bits.uop.robIdx, io.lsin.bits.uop.robIdx) 206 private val SRC_NUM = 8 207 private val Seq( 208 super_rep_idx, fast_rep_idx, lsq_rep_idx, high_pf_idx, 209 int_iss_idx, vec_iss_idx, l2l_fwd_idx, low_pf_idx 210 ) = (0 until SRC_NUM).toSeq 211 // load flow source valid 212 val s0_src_valid_vec = WireInit(VecInit(Seq( 213 io.ldu_io.replay.valid && io.ldu_io.replay.bits.forward_tlDchannel, 214 io.ldu_io.fast_rep_in.valid, 215 io.ldu_io.replay.valid && !io.ldu_io.replay.bits.forward_tlDchannel && !s0_rep_stall, 216 io.ldu_io.prefetch_req.valid && io.ldu_io.prefetch_req.bits.confidence > 0.U, 217 io.lsin.valid, // int flow first issue or software prefetch 218 io.vec_stu_io.in.valid, 219 io.ldu_io.l2l_fwd_in.valid && io.ldu_io.ld_fast_match, 220 io.ldu_io.prefetch_req.valid && io.ldu_io.prefetch_req.bits.confidence === 0.U, 221 ))) 222 // load flow source ready 223 val s0_src_ready_vec = Wire(Vec(SRC_NUM, Bool())) 224 s0_src_ready_vec(0) := true.B 225 for(i <- 1 until SRC_NUM){ 226 s0_src_ready_vec(i) := !s0_src_valid_vec.take(i).reduce(_ || _) 227 } 228 // load flow source select (OH) 229 val s0_src_select_vec = WireInit(VecInit((0 until SRC_NUM).map{i => s0_src_valid_vec(i) && s0_src_ready_vec(i)})) 230 val s0_hw_prf_select = s0_src_select_vec(high_pf_idx) || s0_src_select_vec(low_pf_idx) 231 232 if (backendParams.debugEn){ 233 dontTouch(s0_src_valid_vec) 234 dontTouch(s0_src_ready_vec) 235 dontTouch(s0_src_select_vec) 236 } 237 238 s0_valid := s0_src_valid_vec.reduce(_ || _) && !s0_kill 239 240 // which is S0's out is ready and dcache is ready 241 val s0_try_ptr_chasing = s0_src_select_vec(l2l_fwd_idx) 242 val s0_do_try_ptr_chasing = s0_try_ptr_chasing && s0_can_go && io.ldu_io.dcache.req.ready 243 val s0_ptr_chasing_vaddr = io.ldu_io.l2l_fwd_in.data(5, 0) +& io.ldu_io.ld_fast_imm(5, 0) 244 val s0_ptr_chasing_canceled = WireInit(false.B) 245 s0_kill := s0_ptr_chasing_canceled || (s0_out.uop.robIdx.needFlush(io.redirect) && !s0_try_ptr_chasing) 246 247 // prefetch related ctrl signal 248 val s0_prf = Wire(Bool()) 249 val s0_prf_rd = Wire(Bool()) 250 val s0_prf_wr = Wire(Bool()) 251 val s0_hw_prf = s0_hw_prf_select 252 253 io.canAcceptLowConfPrefetch := s0_src_ready_vec(low_pf_idx) && io.ldu_io.dcache.req.ready 254 io.canAcceptHighConfPrefetch := s0_src_ready_vec(high_pf_idx) && io.ldu_io.dcache.req.ready 255 256 if (StorePrefetchL1Enabled) { 257 s0_dcache_ready := Mux(s0_ld_flow, io.ldu_io.dcache.req.ready, io.stu_io.dcache.req.ready) 258 } else { 259 s0_dcache_ready := Mux(s0_ld_flow, io.ldu_io.dcache.req.ready, true.B) 260 } 261 262 // query DTLB 263 io.tlb.req.valid := s0_valid && s0_dcache_ready 264 io.tlb.req.bits.cmd := Mux(s0_prf, 265 Mux(s0_prf_wr, TlbCmd.write, TlbCmd.read), 266 Mux(s0_ld_flow, TlbCmd.read, TlbCmd.write) 267 ) 268 io.tlb.req.bits.vaddr := Mux(s0_hw_prf_select, io.ldu_io.prefetch_req.bits.paddr, s0_vaddr) 269 io.tlb.req.bits.size := Mux(s0_isvec, io.vec_stu_io.in.bits.alignedType(1, 0), LSUOpType.size(s0_uop.fuOpType)) // may broken if use it in feature 270 io.tlb.req.bits.kill := s0_kill 271 io.tlb.req.bits.memidx.is_ld := s0_ld_flow 272 io.tlb.req.bits.memidx.is_st := !s0_ld_flow 273 io.tlb.req.bits.memidx.idx := s0_uop.lqIdx.value 274 io.tlb.req.bits.debug.robIdx := s0_uop.robIdx 275 io.tlb.req.bits.no_translate := s0_hw_prf_select // hw b.reqetch addr does not need to be translated 276 io.tlb.req.bits.debug.pc := s0_uop.pc 277 io.tlb.req.bits.debug.isFirstIssue := s0_isFirstIssue 278 279 // query DCache 280 // for load 281 io.ldu_io.dcache.req.valid := s0_valid && s0_dcache_ready && s0_ld_flow 282 io.ldu_io.dcache.req.bits.cmd := Mux(s0_prf_rd, MemoryOpConstants.M_PFR, 283 Mux(s0_prf_wr, MemoryOpConstants.M_PFW, MemoryOpConstants.M_XRD)) 284 io.ldu_io.dcache.req.bits.vaddr := s0_vaddr 285 io.ldu_io.dcache.req.bits.mask := s0_mask 286 io.ldu_io.dcache.req.bits.data := DontCare 287 io.ldu_io.dcache.req.bits.isFirstIssue := s0_isFirstIssue 288 io.ldu_io.dcache.req.bits.instrtype := Mux(s0_prf, DCACHE_PREFETCH_SOURCE.U, LOAD_SOURCE.U) 289 io.ldu_io.dcache.req.bits.debug_robIdx := s0_uop.robIdx.value 290 io.ldu_io.dcache.req.bits.replayCarry := s0_rep_carry 291 io.ldu_io.dcache.req.bits.id := DontCare // TODO: update cache meta 292 io.ldu_io.dcache.pf_source := Mux(s0_hw_prf_select, io.ldu_io.prefetch_req.bits.pf_source.value, L1_HW_PREFETCH_NULL) 293 io.ldu_io.dcache.is128Req := is128Bit(io.vec_stu_io.in.bits.alignedType) && io.vec_stu_io.in.valid && s0_src_select_vec(vec_iss_idx) 294 295 // for store 296 io.stu_io.dcache.req.valid := s0_valid && s0_dcache_ready && !s0_ld_flow && !s0_prf 297 io.stu_io.dcache.req.bits.cmd := MemoryOpConstants.M_PFW 298 io.stu_io.dcache.req.bits.vaddr := s0_vaddr 299 io.stu_io.dcache.req.bits.instrtype := Mux(s0_prf, DCACHE_PREFETCH_SOURCE.U, STORE_SOURCE.U) 300 301 // load flow priority mux 302 def fromNullSource() = { 303 s0_vaddr := 0.U 304 s0_mask := 0.U 305 s0_uop := 0.U.asTypeOf(new DynInst) 306 s0_try_l2l := false.B 307 s0_has_rob_entry := false.B 308 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 309 s0_mshrid := 0.U 310 s0_isFirstIssue := false.B 311 s0_fast_rep := false.B 312 s0_ld_rep := false.B 313 s0_l2l_fwd := false.B 314 s0_prf := false.B 315 s0_prf_rd := false.B 316 s0_prf_wr := false.B 317 s0_sched_idx := 0.U 318 } 319 320 def fromFastReplaySource(src: LqWriteBundle) = { 321 s0_vaddr := src.vaddr 322 s0_mask := src.mask 323 s0_uop := src.uop 324 s0_try_l2l := false.B 325 s0_has_rob_entry := src.hasROBEntry 326 s0_rep_carry := src.rep_info.rep_carry 327 s0_mshrid := src.rep_info.mshr_id 328 s0_isFirstIssue := false.B 329 s0_fast_rep := true.B 330 s0_ld_rep := src.isLoadReplay 331 s0_l2l_fwd := false.B 332 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 333 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 334 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 335 s0_sched_idx := src.schedIndex 336 } 337 338 def fromNormalReplaySource(src: LsPipelineBundle) = { 339 s0_vaddr := src.vaddr 340 s0_mask := genVWmask(src.vaddr, src.uop.fuOpType(1, 0)) 341 s0_uop := src.uop 342 s0_try_l2l := false.B 343 s0_has_rob_entry := true.B 344 s0_rep_carry := src.replayCarry 345 s0_mshrid := src.mshrid 346 s0_isFirstIssue := false.B 347 s0_fast_rep := false.B 348 s0_ld_rep := true.B 349 s0_l2l_fwd := false.B 350 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 351 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 352 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 353 s0_sched_idx := src.schedIndex 354 } 355 356 def fromPrefetchSource(src: L1PrefetchReq) = { 357 s0_vaddr := src.getVaddr() 358 s0_mask := 0.U 359 s0_uop := DontCare 360 s0_try_l2l := false.B 361 s0_has_rob_entry := false.B 362 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 363 s0_mshrid := 0.U 364 s0_isFirstIssue := false.B 365 s0_fast_rep := false.B 366 s0_ld_rep := false.B 367 s0_l2l_fwd := false.B 368 s0_prf := true.B 369 s0_prf_rd := !src.is_store 370 s0_prf_wr := src.is_store 371 s0_sched_idx := 0.U 372 } 373 374 def fromIntIssueSource(src: MemExuInput) = { 375 s0_vaddr := src.src(0) + SignExt(src.uop.imm(11, 0), VAddrBits) 376 s0_mask := genVWmask(s0_vaddr, src.uop.fuOpType(1,0)) 377 s0_uop := src.uop 378 s0_try_l2l := false.B 379 s0_has_rob_entry := true.B 380 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 381 s0_mshrid := 0.U 382 s0_isFirstIssue := true.B 383 s0_fast_rep := false.B 384 s0_ld_rep := false.B 385 s0_l2l_fwd := false.B 386 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 387 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 388 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 389 s0_sched_idx := 0.U 390 } 391 392 def fromVecIssueSource(src: VecPipeBundle) = { 393 // For now, vector port handles only vector store flows 394 s0_vaddr := src.vaddr 395 s0_mask := src.mask 396 s0_uop := src.uop 397 s0_try_l2l := false.B 398 s0_has_rob_entry := true.B 399 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 400 s0_mshrid := 0.U 401 // s0_isFirstIssue := src.isFirstIssue 402 s0_fast_rep := false.B 403 s0_ld_rep := false.B 404 s0_l2l_fwd := false.B 405 s0_prf := false.B 406 s0_prf_rd := false.B 407 s0_prf_wr := false.B 408 s0_sched_idx := 0.U 409 410 s0_isvec := true.B 411 s0_vecActive := io.vec_stu_io.in.bits.vecActive 412 // s0_flowPtr := io.vec_stu_io.in.bits.flowPtr 413 // s0_isLastElem := io.vec_stu_io.in.bits.isLastElem 414 } 415 416 def fromLoadToLoadSource(src: LoadToLoadIO) = { 417 s0_vaddr := Cat(src.data(XLEN-1, 6), s0_ptr_chasing_vaddr(5,0)) 418 s0_mask := genVWmask(s0_vaddr, io.ldu_io.ld_fast_fuOpType(1, 0)) 419 // When there's no valid instruction from RS and LSQ, we try the load-to-load forwarding. 420 // Assume the pointer chasing is always ld. 421 s0_uop.fuOpType := io.ldu_io.ld_fast_fuOpType 422 s0_try_l2l := true.B 423 // we dont care s0_isFirstIssue and s0_rsIdx and s0_sqIdx in S0 when trying pointchasing 424 // because these signals will be updated in S1 425 s0_has_rob_entry := false.B 426 s0_mshrid := 0.U 427 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 428 s0_isFirstIssue := true.B 429 s0_fast_rep := false.B 430 s0_ld_rep := false.B 431 s0_l2l_fwd := true.B 432 s0_prf := false.B 433 s0_prf_rd := false.B 434 s0_prf_wr := false.B 435 s0_sched_idx := 0.U 436 } 437 438 // set default 439 s0_uop := DontCare 440 when (s0_src_select_vec(super_rep_idx)) { fromNormalReplaySource(io.ldu_io.replay.bits) } 441 .elsewhen (s0_src_select_vec(fast_rep_idx)) { fromFastReplaySource(io.ldu_io.fast_rep_in.bits) } 442 .elsewhen (s0_src_select_vec(lsq_rep_idx)) { fromNormalReplaySource(io.ldu_io.replay.bits) } 443 .elsewhen (s0_hw_prf_select) { fromPrefetchSource(io.ldu_io.prefetch_req.bits) } 444 .elsewhen (s0_src_select_vec(int_iss_idx)) { fromIntIssueSource(io.lsin.bits) } 445 .elsewhen (s0_src_select_vec(vec_iss_idx)) { fromVecIssueSource(io.vec_stu_io.in.bits) } 446 .otherwise { 447 if (EnableLoadToLoadForward) { 448 fromLoadToLoadSource(io.ldu_io.l2l_fwd_in) 449 } else { 450 fromNullSource() 451 } 452 } 453 454 // address align check 455 val s0_addr_aligned = LookupTree(Mux(s0_isvec, io.vec_stu_io.in.bits.alignedType(1,0), s0_uop.fuOpType(1, 0)), List( 456 "b00".U -> true.B, //b 457 "b01".U -> (s0_vaddr(0) === 0.U), //h 458 "b10".U -> (s0_vaddr(1, 0) === 0.U), //w 459 "b11".U -> (s0_vaddr(2, 0) === 0.U) //d 460 ))// may broken if use it in feature 461 462 // accept load flow if dcache ready (tlb is always ready) 463 // TODO: prefetch need writeback to loadQueueFlag 464 s0_out := DontCare 465 s0_out.vaddr := s0_vaddr 466 s0_out.mask := s0_mask 467 s0_out.uop := s0_uop 468 s0_out.isFirstIssue := s0_isFirstIssue 469 s0_out.hasROBEntry := s0_has_rob_entry 470 s0_out.isPrefetch := s0_prf 471 s0_out.isHWPrefetch := s0_hw_prf 472 s0_out.isFastReplay := s0_fast_rep 473 s0_out.isLoadReplay := s0_ld_rep 474 s0_out.isFastPath := s0_l2l_fwd 475 s0_out.mshrid := s0_mshrid 476 s0_out.isvec := s0_isvec 477 s0_out.isLastElem := s0_isLastElem 478 s0_out.vecActive := s0_vecActive 479 // s0_out.sflowPtr := s0_flowPtr 480 s0_out.uop.exceptionVec(loadAddrMisaligned) := !s0_addr_aligned && s0_ld_flow 481 s0_out.uop.exceptionVec(storeAddrMisaligned) := !s0_addr_aligned && !s0_ld_flow 482 s0_out.forward_tlDchannel := s0_src_select_vec(super_rep_idx) 483 when(io.tlb.req.valid && s0_isFirstIssue) { 484 s0_out.uop.debugInfo.tlbFirstReqTime := GTimer() 485 }.otherwise{ 486 s0_out.uop.debugInfo.tlbFirstReqTime := s0_uop.debugInfo.tlbFirstReqTime 487 } 488 s0_out.schedIndex := s0_sched_idx 489 490 // load fast replay 491 io.ldu_io.fast_rep_in.ready := (s0_can_go && io.ldu_io.dcache.req.ready && s0_src_ready_vec(fast_rep_idx)) 492 493 // load flow source ready 494 // cache missed load has highest priority 495 // always accept cache missed load flow from load replay queue 496 io.ldu_io.replay.ready := (s0_can_go && io.ldu_io.dcache.req.ready && (s0_src_ready_vec(lsq_rep_idx) && !s0_rep_stall || s0_src_select_vec(super_rep_idx))) 497 498 // accept load flow from rs when: 499 // 1) there is no lsq-replayed load 500 // 2) there is no fast replayed load 501 // 3) there is no high confidence prefetch request 502 io.lsin.ready := (s0_can_go && 503 Mux(FuType.isLoad(io.lsin.bits.uop.fuType), io.ldu_io.dcache.req.ready, 504 (if (StorePrefetchL1Enabled) io.stu_io.dcache.req.ready else true.B)) && s0_src_ready_vec(int_iss_idx)) 505 io.vec_stu_io.in.ready := s0_can_go && io.ldu_io.dcache.req.ready && s0_src_ready_vec(vec_iss_idx) 506 507 508 // for hw prefetch load flow feedback, to be added later 509 // io.prefetch_in.ready := s0_hw_prf_select 510 511 // dcache replacement extra info 512 // TODO: should prefetch load update replacement? 513 io.ldu_io.dcache.replacementUpdated := Mux(s0_src_select_vec(lsq_rep_idx) || s0_src_select_vec(super_rep_idx), io.ldu_io.replay.bits.replacementUpdated, false.B) 514 515 io.stu_io.prefetch_req.ready := s1_ready && io.stu_io.dcache.req.ready && !io.lsin.valid 516 517 // load debug 518 XSDebug(io.ldu_io.dcache.req.fire && s0_ld_flow, 519 p"[DCACHE LOAD REQ] pc ${Hexadecimal(s0_uop.pc)}, vaddr ${Hexadecimal(s0_vaddr)}\n" 520 ) 521 XSDebug(s0_valid && s0_ld_flow, 522 p"S0: pc ${Hexadecimal(s0_out.uop.pc)}, lqIdx ${Hexadecimal(s0_out.uop.lqIdx.asUInt)}, " + 523 p"vaddr ${Hexadecimal(s0_out.vaddr)}, mask ${Hexadecimal(s0_out.mask)}\n") 524 525 // store debug 526 XSDebug(io.stu_io.dcache.req.fire && !s0_ld_flow, 527 p"[DCACHE STORE REQ] pc ${Hexadecimal(s0_uop.pc)}, vaddr ${Hexadecimal(s0_vaddr)}\n" 528 ) 529 XSDebug(s0_valid && !s0_ld_flow, 530 p"S0: pc ${Hexadecimal(s0_out.uop.pc)}, sqIdx ${Hexadecimal(s0_out.uop.sqIdx.asUInt)}, " + 531 p"vaddr ${Hexadecimal(s0_out.vaddr)}, mask ${Hexadecimal(s0_out.mask)}\n") 532 533 534 // Pipeline 535 // -------------------------------------------------------------------------------- 536 // stage 1 537 // -------------------------------------------------------------------------------- 538 // TLB resp (send paddr to dcache) 539 val s1_valid = RegInit(false.B) 540 val s1_in = Wire(new LqWriteBundle) 541 val s1_out = Wire(new LqWriteBundle) 542 val s1_kill = Wire(Bool()) 543 val s1_can_go = s2_ready 544 val s1_fire = s1_valid && !s1_kill && s1_can_go 545 val s1_ld_flow = RegNext(s0_ld_flow) 546 val s1_isvec = RegEnable(s0_out.isvec, false.B, s0_fire) 547 val s1_isLastElem = RegEnable(s0_out.isLastElem, false.B, s0_fire) 548 549 s1_ready := !s1_valid || s1_kill || s2_ready 550 when (s0_fire) { s1_valid := true.B } 551 .elsewhen (s1_fire) { s1_valid := false.B } 552 .elsewhen (s1_kill) { s1_valid := false.B } 553 s1_in := RegEnable(s0_out, s0_fire) 554 555 val s1_fast_rep_dly_err = RegNext(io.ldu_io.fast_rep_in.bits.delayedLoadError) 556 val s1_fast_rep_kill = s1_fast_rep_dly_err && s1_in.isFastReplay 557 val s1_l2l_fwd_dly_err = RegNext(io.ldu_io.l2l_fwd_in.dly_ld_err) 558 val s1_l2l_fwd_kill = s1_l2l_fwd_dly_err && s1_in.isFastPath 559 val s1_late_kill = s1_fast_rep_kill || s1_l2l_fwd_kill 560 val s1_vaddr_hi = Wire(UInt()) 561 val s1_vaddr_lo = Wire(UInt()) 562 val s1_vaddr = Wire(UInt()) 563 val s1_paddr_dup_lsu = Wire(UInt()) 564 val s1_paddr_dup_dcache = Wire(UInt()) 565 val s1_ld_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, LduCfg).asUInt.orR // af & pf exception were modified below. 566 val s1_st_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, StaCfg).asUInt.orR // af & pf exception were modified below. 567 val s1_exception = (s1_ld_flow && s1_ld_exception) || (!s1_ld_flow && s1_st_exception) 568 val s1_tlb_miss = io.tlb.resp.bits.miss 569 val s1_prf = s1_in.isPrefetch 570 val s1_hw_prf = s1_in.isHWPrefetch 571 val s1_sw_prf = s1_prf && !s1_hw_prf 572 val s1_tlb_memidx = io.tlb.resp.bits.memidx 573 574 // mmio cbo decoder 575 val s1_mmio_cbo = (s1_in.uop.fuOpType === LSUOpType.cbo_clean || 576 s1_in.uop.fuOpType === LSUOpType.cbo_flush || 577 s1_in.uop.fuOpType === LSUOpType.cbo_inval) && !s1_ld_flow && !s1_prf 578 val s1_mmio = s1_mmio_cbo 579 580 s1_vaddr_hi := s1_in.vaddr(VAddrBits - 1, 6) 581 s1_vaddr_lo := s1_in.vaddr(5, 0) 582 s1_vaddr := Cat(s1_vaddr_hi, s1_vaddr_lo) 583 s1_paddr_dup_lsu := io.tlb.resp.bits.paddr(0) 584 s1_paddr_dup_dcache := io.tlb.resp.bits.paddr(1) 585 586 when (s1_tlb_memidx.is_ld && io.tlb.resp.valid && !s1_tlb_miss && 587 s1_tlb_memidx.idx === s1_in.uop.lqIdx.value && s1_ld_flow) { 588 // printf("Load idx = %d\n", s1_tlb_memidx.idx) 589 s1_out.uop.debugInfo.tlbRespTime := GTimer() 590 } .elsewhen(s1_tlb_memidx.is_st && io.tlb.resp.valid && !s1_tlb_miss && 591 s1_tlb_memidx.idx === s1_out.uop.sqIdx.value && !s1_ld_flow) { 592 // printf("Store idx = %d\n", s1_tlb_memidx.idx) 593 s1_out.uop.debugInfo.tlbRespTime := GTimer() 594 } 595 596 io.tlb.req_kill := s1_kill 597 io.tlb.resp.ready := true.B 598 599 io.ldu_io.dcache.s1_paddr_dup_lsu <> s1_paddr_dup_lsu 600 io.ldu_io.dcache.s1_paddr_dup_dcache <> s1_paddr_dup_dcache 601 io.ldu_io.dcache.s1_kill := s1_kill || s1_tlb_miss || s1_exception 602 io.ldu_io.dcache.s1_kill_data_read := s1_kill || s1_tlb_miss 603 604 // store to load forwarding 605 io.ldu_io.sbuffer.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_fast_rep_kill || s1_prf || !s1_ld_flow) 606 io.ldu_io.sbuffer.vaddr := s1_vaddr 607 io.ldu_io.sbuffer.paddr := s1_paddr_dup_lsu 608 io.ldu_io.sbuffer.uop := s1_in.uop 609 io.ldu_io.sbuffer.sqIdx := s1_in.uop.sqIdx 610 io.ldu_io.sbuffer.mask := s1_in.mask 611 io.ldu_io.sbuffer.pc := s1_in.uop.pc // FIXME: remove it 612 613 io.ldu_io.ubuffer.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_fast_rep_kill || s1_prf || !s1_ld_flow) 614 io.ldu_io.ubuffer.vaddr := s1_vaddr 615 io.ldu_io.ubuffer.paddr := s1_paddr_dup_lsu 616 io.ldu_io.ubuffer.uop := s1_in.uop 617 io.ldu_io.ubuffer.sqIdx := s1_in.uop.sqIdx 618 io.ldu_io.ubuffer.mask := s1_in.mask 619 io.ldu_io.ubuffer.pc := s1_in.uop.pc // FIXME: remove it 620 621 io.ldu_io.vec_forward.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_fast_rep_kill || s1_prf || !s1_ld_flow) 622 io.ldu_io.vec_forward.vaddr := s1_vaddr 623 io.ldu_io.vec_forward.paddr := s1_paddr_dup_lsu 624 io.ldu_io.vec_forward.uop := s1_in.uop 625 io.ldu_io.vec_forward.sqIdx := s1_in.uop.sqIdx 626 io.ldu_io.vec_forward.mask := s1_in.mask 627 io.ldu_io.vec_forward.pc := s1_in.uop.pc // FIXME: remove it 628 629 io.ldu_io.lsq.forward.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_fast_rep_kill || s1_prf || !s1_ld_flow) 630 io.ldu_io.lsq.forward.vaddr := s1_vaddr 631 io.ldu_io.lsq.forward.paddr := s1_paddr_dup_lsu 632 io.ldu_io.lsq.forward.uop := s1_in.uop 633 io.ldu_io.lsq.forward.sqIdx := s1_in.uop.sqIdx 634 io.ldu_io.lsq.forward.sqIdxMask := 0.U 635 io.ldu_io.lsq.forward.mask := s1_in.mask 636 io.ldu_io.lsq.forward.pc := s1_in.uop.pc // FIXME: remove it 637 638 // st-ld violation query 639 val s1_nuke = VecInit((0 until StorePipelineWidth).map(w => { 640 io.ldu_io.stld_nuke_query(w).valid && // query valid 641 isAfter(s1_in.uop.robIdx, io.ldu_io.stld_nuke_query(w).bits.robIdx) && // older store 642 // TODO: Fix me when vector instruction 643 (s1_paddr_dup_lsu(PAddrBits-1, 3) === io.ldu_io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 644 (s1_in.mask & io.ldu_io.stld_nuke_query(w).bits.mask).orR // data mask contain 645 })).asUInt.orR && !s1_tlb_miss && s1_ld_flow 646 647 s1_out := s1_in 648 s1_out.vaddr := s1_vaddr 649 s1_out.paddr := s1_paddr_dup_lsu 650 s1_out.tlbMiss := s1_tlb_miss 651 s1_out.ptwBack := io.tlb.resp.bits.ptwBack 652 s1_out.rep_info.debug := s1_in.uop.debugInfo 653 s1_out.rep_info.nuke := s1_nuke && !s1_sw_prf 654 s1_out.lateKill := s1_late_kill 655 656 // store trigger 657 val storeTrigger = Module(new MemTrigger(MemType.STORE)) 658 storeTrigger.io.fromCsrTrigger.tdataVec := io.fromCsrTrigger.tdataVec 659 storeTrigger.io.fromCsrTrigger.tEnableVec := io.fromCsrTrigger.tEnableVec 660 storeTrigger.io.fromCsrTrigger.triggerCanRaiseBpExp := io.fromCsrTrigger.triggerCanRaiseBpExp 661 storeTrigger.io.fromCsrTrigger.debugMode := io.fromCsrTrigger.debugMode 662 storeTrigger.io.fromLoadStore.vaddr := s1_vaddr 663 storeTrigger.io.fromLoadStore.isVectorUnitStride := s1_in.isvec && s1_in.is128bit 664 storeTrigger.io.fromLoadStore.mask := s1_in.mask 665 666 when (s1_ld_flow) { 667 when (!s1_late_kill) { 668 // current ori test will cause the case of ldest == 0, below will be modifeid in the future. 669 // af & pf exception were modified 670 s1_out.uop.exceptionVec(loadPageFault) := io.tlb.resp.bits.excp(0).pf.ld 671 s1_out.uop.exceptionVec(loadGuestPageFault) := io.tlb.resp.bits.excp(0).gpf.ld 672 s1_out.uop.exceptionVec(loadAccessFault) := io.tlb.resp.bits.excp(0).af.ld 673 } .otherwise { 674 s1_out.uop.exceptionVec(loadAddrMisaligned) := false.B 675 s1_out.uop.exceptionVec(loadAccessFault) := s1_late_kill 676 } 677 } .otherwise { 678 s1_out.uop.exceptionVec(storePageFault) := io.tlb.resp.bits.excp(0).pf.st 679 s1_out.uop.exceptionVec(storeGuestPageFault) := io.tlb.resp.bits.excp(0).gpf.st 680 s1_out.uop.exceptionVec(storeAccessFault) := io.tlb.resp.bits.excp(0).af.st 681 s1_out.uop.trigger := storeTrigger.io.toLoadStore.triggerAction 682 s1_out.uop.exceptionVec(breakPoint) := TriggerAction.isExp(storeTrigger.io.toLoadStore.triggerAction) 683 } 684 685 // load trigger 686 val loadTrigger = Module(new MemTrigger(MemType.LOAD)) 687 loadTrigger.io.fromCsrTrigger.tdataVec := io.fromCsrTrigger.tdataVec 688 loadTrigger.io.fromCsrTrigger.tEnableVec := io.fromCsrTrigger.tEnableVec 689 loadTrigger.io.fromCsrTrigger.triggerCanRaiseBpExp := io.fromCsrTrigger.triggerCanRaiseBpExp 690 loadTrigger.io.fromCsrTrigger.debugMode := io.fromCsrTrigger.debugMode 691 loadTrigger.io.fromLoadStore.vaddr := s1_vaddr 692 loadTrigger.io.fromLoadStore.isVectorUnitStride := s1_in.isvec && s1_in.is128bit 693 loadTrigger.io.fromLoadStore.mask := s1_in.mask 694 695 when (s1_ld_flow) { 696 s1_out.uop.exceptionVec(breakPoint) := TriggerAction.isExp(loadTrigger.io.toLoadStore.triggerAction) 697 s1_out.uop.trigger := loadTrigger.io.toLoadStore.triggerAction 698 } 699 700 // pointer chasing 701 val s1_try_ptr_chasing = RegNext(s0_do_try_ptr_chasing, false.B) 702 val s1_ptr_chasing_vaddr = RegEnable(s0_ptr_chasing_vaddr, s0_do_try_ptr_chasing) 703 val s1_fu_op_type_not_ld = WireInit(false.B) 704 val s1_not_fast_match = WireInit(false.B) 705 val s1_addr_mismatch = WireInit(false.B) 706 val s1_addr_misaligned = WireInit(false.B) 707 val s1_ptr_chasing_canceled = WireInit(false.B) 708 val s1_cancel_ptr_chasing = WireInit(false.B) 709 710 s1_kill := s1_late_kill || 711 s1_cancel_ptr_chasing || 712 s1_in.uop.robIdx.needFlush(io.redirect) || 713 RegEnable(s0_kill, false.B, io.lsin.valid || io.ldu_io.replay.valid || io.ldu_io.l2l_fwd_in.valid || io.ldu_io.fast_rep_in.valid || io.vec_stu_io.in.valid) 714 715 if (EnableLoadToLoadForward) { 716 // Sometimes, we need to cancel the load-load forwarding. 717 // These can be put at S0 if timing is bad at S1. 718 // Case 0: CACHE_SET(base + offset) != CACHE_SET(base) (lowest 6-bit addition has an overflow) 719 s1_addr_mismatch := s1_ptr_chasing_vaddr(6) || RegEnable(io.ldu_io.ld_fast_imm(11, 6).orR, s0_do_try_ptr_chasing) 720 // Case 1: the address is misaligned, kill s1 721 s1_addr_misaligned := LookupTree(s1_in.uop.fuOpType(1, 0), List( 722 "b00".U -> false.B, //b 723 "b01".U -> (s1_vaddr(0) =/= 0.U), //h 724 "b10".U -> (s1_vaddr(1, 0) =/= 0.U), //w 725 "b11".U -> (s1_vaddr(2, 0) =/= 0.U) //d 726 )) 727 // Case 2: this load-load uop is cancelled 728 s1_ptr_chasing_canceled := !io.lsin.valid || FuType.isStore(io.lsin.bits.uop.fuType) 729 730 when (s1_try_ptr_chasing) { 731 s1_cancel_ptr_chasing := s1_addr_mismatch || s1_addr_misaligned || s1_ptr_chasing_canceled 732 733 s1_in.uop := io.lsin.bits.uop 734 s1_in.isFirstIssue := io.lsin.bits.isFirstIssue 735 s1_vaddr_lo := s1_ptr_chasing_vaddr(5, 0) 736 s1_paddr_dup_lsu := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 737 s1_paddr_dup_dcache := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 738 739 // recored tlb time when get the data to ensure the correctness of the latency calculation (although it should not record in here, because it does not use tlb) 740 s1_in.uop.debugInfo.tlbFirstReqTime := GTimer() 741 s1_in.uop.debugInfo.tlbRespTime := GTimer() 742 } 743 when (!s1_cancel_ptr_chasing) { 744 s0_ptr_chasing_canceled := s1_try_ptr_chasing && !io.ldu_io.replay.fire && !io.ldu_io.fast_rep_in.fire && !(s0_src_valid_vec(high_pf_idx) && io.canAcceptHighConfPrefetch) 745 when (s1_try_ptr_chasing) { 746 io.lsin.ready := true.B 747 } 748 } 749 } 750 751 // pre-calcuate sqIdx mask in s0, then send it to lsq in s1 for forwarding 752 val s1_sqIdx_mask = RegNext(UIntToMask(s0_out.uop.sqIdx.value, StoreQueueSize)) 753 // to enable load-load, sqIdxMask must be calculated based on lsin.uop 754 // If the timing here is not OK, load-load forwarding has to be disabled. 755 // Or we calculate sqIdxMask at RS?? 756 io.ldu_io.lsq.forward.sqIdxMask := s1_sqIdx_mask 757 if (EnableLoadToLoadForward) { 758 when (s1_try_ptr_chasing) { 759 io.ldu_io.lsq.forward.sqIdxMask := UIntToMask(io.lsin.bits.uop.sqIdx.value, StoreQueueSize) 760 } 761 } 762 763 io.ldu_io.forward_mshr.valid := s1_valid && s1_out.forward_tlDchannel && s1_ld_flow 764 io.ldu_io.forward_mshr.mshrid := s1_out.mshrid 765 io.ldu_io.forward_mshr.paddr := s1_out.paddr 766 767 io.ldu_io.wakeup.valid := s0_fire && s0_ld_flow && (s0_src_select_vec(super_rep_idx) || s0_src_select_vec(fast_rep_idx) || s0_src_select_vec(lsq_rep_idx) || s0_src_select_vec(int_iss_idx)) 768 io.ldu_io.wakeup.bits := s0_uop 769 770 io.stu_io.dcache.s1_kill := s1_tlb_miss || s1_exception || s1_mmio || s1_in.uop.robIdx.needFlush(io.redirect) 771 io.stu_io.dcache.s1_paddr := s1_paddr_dup_dcache 772 773 774 // load debug 775 XSDebug(s1_valid && s1_ld_flow, 776 p"S1: pc ${Hexadecimal(s1_out.uop.pc)}, lId ${Hexadecimal(s1_out.uop.lqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 777 p"paddr ${Hexadecimal(s1_out.paddr)}, mmio ${s1_out.mmio}\n") 778 779 // store debug 780 XSDebug(s1_valid && !s1_ld_flow, 781 p"S1: pc ${Hexadecimal(s1_out.uop.pc)}, lId ${Hexadecimal(s1_out.uop.sqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 782 p"paddr ${Hexadecimal(s1_out.paddr)}, mmio ${s1_out.mmio}\n") 783 784 // store out 785 io.stu_io.lsq.valid := s1_valid && !s1_ld_flow && !s1_prf && !s1_isvec 786 io.stu_io.lsq.bits := s1_out 787 io.stu_io.lsq.bits.miss := s1_tlb_miss 788 789 io.vec_stu_io.lsq.valid := s1_valid && !s1_ld_flow && !s1_prf && s1_isvec 790 io.vec_stu_io.lsq.bits := s1_out 791 io.vec_stu_io.lsq.bits.miss := s1_tlb_miss 792 io.vec_stu_io.lsq.bits.isLastElem := s1_isLastElem 793 794 io.stu_io.st_mask_out.valid := s1_valid && !s1_ld_flow && !s1_prf 795 io.stu_io.st_mask_out.bits.mask := s1_out.mask 796 io.stu_io.st_mask_out.bits.sqIdx := s1_out.uop.sqIdx 797 798 io.stu_io.issue.valid := s1_valid && !s1_tlb_miss && !s1_ld_flow && !s1_prf && !s1_isvec 799 io.stu_io.issue.bits := RegEnable(io.lsin.bits, io.lsin.fire) 800 801 // st-ld violation dectect request 802 io.stu_io.stld_nuke_query.valid := s1_valid && !s1_tlb_miss && !s1_ld_flow && !s1_prf 803 io.stu_io.stld_nuke_query.bits.robIdx := s1_in.uop.robIdx 804 io.stu_io.stld_nuke_query.bits.paddr := s1_paddr_dup_lsu 805 io.stu_io.stld_nuke_query.bits.mask := s1_in.mask 806 807 // Pipeline 808 // -------------------------------------------------------------------------------- 809 // stage 2 810 // -------------------------------------------------------------------------------- 811 // s2: DCache resp 812 val s2_valid = RegInit(false.B) 813 val s2_in = Wire(new LqWriteBundle) 814 val s2_out = Wire(new LqWriteBundle) 815 val s2_kill = Wire(Bool()) 816 val s2_can_go = s3_ready 817 val s2_fire = s2_valid && !s2_kill && s2_can_go 818 val s2_isvec = RegEnable(s1_isvec, false.B, s1_fire) 819 val s2_vecActive = RegEnable(s1_out.vecActive, true.B, s1_fire) 820 val s2_paddr = RegEnable(s1_paddr_dup_lsu, s1_fire) 821 822 s2_kill := s2_in.uop.robIdx.needFlush(io.redirect) 823 s2_ready := !s2_valid || s2_kill || s3_ready 824 when (s1_fire) { s2_valid := true.B } 825 .elsewhen (s2_fire) { s2_valid := false.B } 826 .elsewhen (s2_kill) { s2_valid := false.B } 827 s2_in := RegEnable(s1_out, s1_fire) 828 829 val s2_pmp = WireInit(io.pmp) 830 831 val s2_prf = s2_in.isPrefetch 832 val s2_hw_prf = s2_in.isHWPrefetch 833 val s2_ld_flow = RegEnable(s1_ld_flow, s1_fire) 834 835 // exception that may cause load addr to be invalid / illegal 836 // if such exception happen, that inst and its exception info 837 // will be force writebacked to rob 838 val s2_exception_vec = WireInit(s2_in.uop.exceptionVec) 839 when (s2_ld_flow) { 840 when (!s2_in.lateKill) { 841 s2_exception_vec(loadAccessFault) := s2_vecActive && ( 842 s2_in.uop.exceptionVec(loadAccessFault) || s2_pmp.ld || 843 s2_fwd_frm_d_chan && s2_d_corrupt || 844 s2_fwd_frm_mshr && s2_mshr_corrupt 845 ) 846 // soft prefetch will not trigger any exception (but ecc error interrupt may be triggered) 847 when (s2_prf || s2_in.tlbMiss) { 848 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 849 } 850 } 851 } .otherwise { 852 s2_exception_vec(storeAccessFault) := s2_in.uop.exceptionVec(storeAccessFault) || s2_pmp.st 853 when (s2_prf || s2_in.tlbMiss) { 854 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 855 } 856 } 857 val s2_ld_exception = ExceptionNO.selectByFu(s2_exception_vec, LduCfg).asUInt.orR && s2_ld_flow 858 val s2_st_exception = ExceptionNO.selectByFu(s2_exception_vec, StaCfg).asUInt.orR && !s2_ld_flow 859 val s2_exception = s2_ld_exception || s2_st_exception 860 861 val (s2_fwd_frm_d_chan, s2_fwd_data_frm_d_chan, s2_d_corrupt) = io.ldu_io.tl_d_channel.forward(s1_valid && s1_out.forward_tlDchannel, s1_out.mshrid, s1_out.paddr) 862 val (s2_fwd_data_valid, s2_fwd_frm_mshr, s2_fwd_data_frm_mshr, s2_mshr_corrupt) = io.ldu_io.forward_mshr.forward() 863 val s2_fwd_frm_d_chan_or_mshr = s2_fwd_data_valid && (s2_fwd_frm_d_chan || s2_fwd_frm_mshr) 864 865 // writeback access fault caused by ecc error / bus error 866 // * ecc data error is slow to generate, so we will not use it until load stage 3 867 // * in load stage 3, an extra signal io.load_error will be used to 868 val s2_actually_mmio = s2_pmp.mmio 869 val s2_ld_mmio = !s2_prf && 870 s2_actually_mmio && 871 !s2_exception && 872 !s2_in.tlbMiss && 873 s2_ld_flow 874 val s2_st_mmio = !s2_prf && 875 (RegNext(s1_mmio) || s2_pmp.mmio) && 876 !s2_exception && 877 !s2_in.tlbMiss && 878 !s2_ld_flow 879 val s2_st_atomic = !s2_prf && 880 (RegNext(s1_mmio) || s2_pmp.atomic) && 881 !s2_exception && 882 !s2_in.tlbMiss && 883 !s2_ld_flow 884 val s2_full_fwd = Wire(Bool()) 885 val s2_mem_amb = s2_in.uop.storeSetHit && 886 io.ldu_io.lsq.forward.addrInvalid 887 888 val s2_tlb_miss = s2_in.tlbMiss 889 val s2_fwd_fail = io.ldu_io.lsq.forward.dataInvalid || io.ldu_io.vec_forward.dataInvalid 890 val s2_dcache_miss = io.ldu_io.dcache.resp.bits.miss && 891 !s2_fwd_frm_d_chan_or_mshr && 892 !s2_full_fwd 893 894 val s2_mq_nack = io.ldu_io.dcache.s2_mq_nack && 895 !s2_fwd_frm_d_chan_or_mshr && 896 !s2_full_fwd 897 898 val s2_bank_conflict = io.ldu_io.dcache.s2_bank_conflict && 899 !s2_fwd_frm_d_chan_or_mshr && 900 !s2_full_fwd 901 902 val s2_wpu_pred_fail = io.ldu_io.dcache.s2_wpu_pred_fail && 903 !s2_fwd_frm_d_chan_or_mshr && 904 !s2_full_fwd 905 906 val s2_rar_nack = io.ldu_io.lsq.ldld_nuke_query.req.valid && 907 !io.ldu_io.lsq.ldld_nuke_query.req.ready 908 909 val s2_raw_nack = io.ldu_io.lsq.stld_nuke_query.req.valid && 910 !io.ldu_io.lsq.stld_nuke_query.req.ready 911 912 // st-ld violation query 913 // NeedFastRecovery Valid when 914 // 1. Fast recovery query request Valid. 915 // 2. Load instruction is younger than requestors(store instructions). 916 // 3. Physical address match. 917 // 4. Data contains. 918 val s2_nuke = VecInit((0 until StorePipelineWidth).map(w => { 919 io.ldu_io.stld_nuke_query(w).valid && // query valid 920 isAfter(s2_in.uop.robIdx, io.ldu_io.stld_nuke_query(w).bits.robIdx) && // older store 921 // TODO: Fix me when vector instruction 922 (s2_in.paddr(PAddrBits-1, 3) === io.ldu_io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 923 (s2_in.mask & io.ldu_io.stld_nuke_query(w).bits.mask).orR // data mask contain 924 })).asUInt.orR && s2_ld_flow || s2_in.rep_info.nuke 925 926 val s2_cache_handled = io.ldu_io.dcache.resp.bits.handled 927 val s2_cache_tag_error = RegNext(io.csrCtrl.cache_error_enable) && 928 io.ldu_io.dcache.resp.bits.tag_error 929 930 val s2_troublem = !s2_exception && 931 !s2_ld_mmio && 932 !s2_prf && 933 !s2_in.lateKill && 934 s2_ld_flow 935 936 io.ldu_io.dcache.resp.ready := true.B 937 io.stu_io.dcache.resp.ready := true.B 938 val s2_dcache_should_resp = !(s2_in.tlbMiss || s2_exception || s2_ld_mmio || s2_prf || s2_in.lateKill) && s2_ld_flow 939 assert(!(s2_valid && (s2_dcache_should_resp && !io.ldu_io.dcache.resp.valid)), "DCache response got lost") 940 941 // fast replay require 942 val s2_dcache_fast_rep = (s2_mq_nack || !s2_dcache_miss && (s2_bank_conflict || s2_wpu_pred_fail)) 943 val s2_nuke_fast_rep = !s2_mq_nack && 944 !s2_dcache_miss && 945 !s2_bank_conflict && 946 !s2_wpu_pred_fail && 947 !s2_rar_nack && 948 !s2_raw_nack && 949 s2_nuke 950 951 val s2_fast_rep = !s2_mem_amb && 952 !s2_tlb_miss && 953 !s2_fwd_fail && 954 (s2_dcache_fast_rep || s2_nuke_fast_rep) && 955 s2_troublem 956 957 // need allocate new entry 958 val s2_can_query = !s2_mem_amb && 959 !s2_tlb_miss && 960 !s2_fwd_fail && 961 !s2_dcache_fast_rep && 962 s2_troublem 963 964 val s2_data_fwded = s2_dcache_miss && (s2_full_fwd || s2_cache_tag_error) 965 966 // ld-ld violation require 967 io.ldu_io.lsq.ldld_nuke_query.req.valid := s2_valid && s2_can_query 968 io.ldu_io.lsq.ldld_nuke_query.req.bits.uop := s2_in.uop 969 io.ldu_io.lsq.ldld_nuke_query.req.bits.mask := s2_in.mask 970 io.ldu_io.lsq.ldld_nuke_query.req.bits.paddr := s2_in.paddr 971 io.ldu_io.lsq.ldld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid, true.B, !s2_dcache_miss) 972 973 // st-ld violation require 974 io.ldu_io.lsq.stld_nuke_query.req.valid := s2_valid && s2_can_query 975 io.ldu_io.lsq.stld_nuke_query.req.bits.uop := s2_in.uop 976 io.ldu_io.lsq.stld_nuke_query.req.bits.mask := s2_in.mask 977 io.ldu_io.lsq.stld_nuke_query.req.bits.paddr := s2_in.paddr 978 io.ldu_io.lsq.stld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid, true.B, !s2_dcache_miss) 979 980 // merge forward result 981 // lsq has higher priority than sbuffer 982 val s2_fwd_mask = Wire(Vec((VLEN/8), Bool())) 983 val s2_fwd_data = Wire(Vec((VLEN/8), UInt(8.W))) 984 s2_full_fwd := ((~s2_fwd_mask.asUInt).asUInt & s2_in.mask) === 0.U && !io.ldu_io.lsq.forward.dataInvalid && !io.ldu_io.vec_forward.dataInvalid 985 // generate XLEN/8 Muxs 986 for (i <- 0 until VLEN / 8) { 987 s2_fwd_mask(i) := io.ldu_io.lsq.forward.forwardMask(i) || io.ldu_io.sbuffer.forwardMask(i) || io.ldu_io.vec_forward.forwardMask(i) || io.ldu_io.ubuffer.forwardMask(i) 988 s2_fwd_data(i) := 989 Mux(io.ldu_io.lsq.forward.forwardMask(i), io.ldu_io.lsq.forward.forwardData(i), 990 Mux(io.ldu_io.vec_forward.forwardMask(i), io.ldu_io.vec_forward.forwardData(i), 991 Mux(io.ldu_io.ubuffer.forwardMask(i), io.ldu_io.ubuffer.forwardData(i), 992 io.ldu_io.sbuffer.forwardData(i)))) 993 } 994 995 XSDebug(s2_fire && s2_ld_flow, "[FWD LOAD RESP] pc %x fwd %x(%b) + %x(%b)\n", 996 s2_in.uop.pc, 997 io.ldu_io.lsq.forward.forwardData.asUInt, io.ldu_io.lsq.forward.forwardMask.asUInt, 998 s2_in.forwardData.asUInt, s2_in.forwardMask.asUInt 999 ) 1000 1001 // 1002 s2_out := s2_in 1003 s2_out.data := 0.U // data will be generated in load s3 1004 s2_out.uop.fpWen := s2_in.uop.fpWen && !s2_exception && s2_ld_flow 1005 s2_out.mmio := s2_ld_mmio || s2_st_mmio 1006 s2_out.atomic := s2_st_atomic 1007 s2_out.uop.flushPipe := false.B 1008 s2_out.uop.exceptionVec := s2_exception_vec 1009 s2_out.forwardMask := s2_fwd_mask 1010 s2_out.forwardData := s2_fwd_data 1011 s2_out.handledByMSHR := s2_cache_handled 1012 s2_out.miss := s2_dcache_miss && s2_troublem 1013 s2_out.feedbacked := io.feedback_fast.valid && !io.feedback_fast.bits.hit 1014 1015 // Generate replay signal caused by: 1016 // * st-ld violation check 1017 // * tlb miss 1018 // * dcache replay 1019 // * forward data invalid 1020 // * dcache miss 1021 s2_out.rep_info.mem_amb := s2_mem_amb && s2_troublem 1022 s2_out.rep_info.tlb_miss := s2_tlb_miss && s2_troublem 1023 s2_out.rep_info.fwd_fail := s2_fwd_fail && s2_troublem 1024 s2_out.rep_info.dcache_rep := s2_mq_nack && s2_troublem 1025 s2_out.rep_info.dcache_miss := s2_dcache_miss && s2_troublem 1026 s2_out.rep_info.bank_conflict := s2_bank_conflict && s2_troublem 1027 s2_out.rep_info.wpu_fail := s2_wpu_pred_fail && s2_troublem 1028 s2_out.rep_info.rar_nack := s2_rar_nack && s2_troublem 1029 s2_out.rep_info.raw_nack := s2_raw_nack && s2_troublem 1030 s2_out.rep_info.nuke := s2_nuke && s2_troublem 1031 s2_out.rep_info.full_fwd := s2_data_fwded 1032 s2_out.rep_info.data_inv_sq_idx := Mux(io.ldu_io.vec_forward.dataInvalid, s2_out.uop.sqIdx, io.ldu_io.lsq.forward.dataInvalidSqIdx) 1033 s2_out.rep_info.addr_inv_sq_idx := Mux(io.ldu_io.vec_forward.addrInvalid, s2_out.uop.sqIdx, io.ldu_io.lsq.forward.addrInvalidSqIdx) 1034 s2_out.rep_info.rep_carry := io.ldu_io.dcache.resp.bits.replayCarry 1035 s2_out.rep_info.mshr_id := io.ldu_io.dcache.resp.bits.mshr_id 1036 s2_out.rep_info.last_beat := s2_in.paddr(log2Up(refillBytes)) 1037 s2_out.rep_info.debug := s2_in.uop.debugInfo 1038 s2_out.rep_info.tlb_id := io.ldu_io.tlb_hint.id 1039 s2_out.rep_info.tlb_full := io.ldu_io.tlb_hint.full 1040 1041 // if forward fail, replay this inst from fetch 1042 val debug_fwd_fail_rep = s2_fwd_fail && !s2_troublem && !s2_in.tlbMiss 1043 // if ld-ld violation is detected, replay from this inst from fetch 1044 val debug_ldld_nuke_rep = false.B // s2_ldld_violation && !s2_ld_mmio && !s2_is_prefetch && !s2_in.tlbMiss 1045 // io.out.bits.uop.replayInst := false.B 1046 1047 // to be removed 1048 val s2_ld_need_fb = !s2_in.isLoadReplay && // already feedbacked 1049 io.ldu_io.lq_rep_full && // LoadQueueReplay is full 1050 s2_out.rep_info.need_rep && // need replay 1051 !s2_exception && // no exception is triggered 1052 !s2_hw_prf && // not hardware prefetch 1053 !s2_isvec 1054 val s2_st_need_fb = !s2_ld_flow && !s2_hw_prf && !s2_isvec 1055 io.feedback_fast.valid := s2_valid && (s2_ld_need_fb || s2_st_need_fb) 1056 io.feedback_fast.bits.hit := Mux(s2_ld_flow, false.B, !s2_tlb_miss) 1057 io.feedback_fast.bits.flushState := s2_in.ptwBack 1058 io.feedback_fast.bits.robIdx := s2_in.uop.robIdx 1059 io.feedback_fast.bits.sourceType := Mux(s2_ld_flow, RSFeedbackType.lrqFull, RSFeedbackType.tlbMiss) 1060 io.feedback_fast.bits.dataInvalidSqIdx := DontCare 1061 1062 val s2_vec_feedback = Wire(Valid(new VSFQFeedback)) 1063 s2_vec_feedback.valid := s2_valid && !s2_ld_flow && !s2_hw_prf && s2_isvec 1064 // s2_vec_feedback.bits.flowPtr := s2_out.sflowPtr 1065 s2_vec_feedback.bits.hit := !s2_tlb_miss 1066 s2_vec_feedback.bits.sourceType := RSFeedbackType.tlbMiss 1067 s2_vec_feedback.bits.paddr := s2_paddr 1068 s2_vec_feedback.bits.mmio := s2_st_mmio 1069 s2_vec_feedback.bits.atomic := s2_st_mmio 1070 s2_vec_feedback.bits.exceptionVec := s2_exception_vec 1071 1072 io.stu_io.lsq_replenish := s2_out 1073 io.stu_io.lsq_replenish.miss := io.ldu_io.dcache.resp.fire && io.ldu_io.dcache.resp.bits.miss 1074 1075 io.ldu_io.ldCancel.ld1Cancel := false.B 1076 1077 // fast wakeup 1078 io.ldu_io.fast_uop.valid := RegNext( 1079 !io.ldu_io.dcache.s1_disable_fast_wakeup && 1080 s1_valid && 1081 !s1_kill && 1082 !io.tlb.resp.bits.miss && 1083 !io.ldu_io.lsq.forward.dataInvalidFast 1084 ) && (s2_valid && !s2_out.rep_info.need_rep && !s2_ld_mmio && s2_ld_flow) && !s2_isvec 1085 io.ldu_io.fast_uop.bits := RegNext(s1_out.uop) 1086 1087 // 1088 io.ldu_io.s2_ptr_chasing := RegEnable(s1_try_ptr_chasing && !s1_cancel_ptr_chasing, false.B, s1_fire) 1089 1090 // prefetch train 1091 io.s0_prefetch_spec := s0_fire 1092 io.s1_prefetch_spec := s1_fire 1093 io.prefetch_train.valid := s2_valid && !s2_actually_mmio && !s2_in.tlbMiss 1094 io.prefetch_train.bits.fromLsPipelineBundle(s2_in) 1095 io.prefetch_train.bits.miss := Mux(s2_ld_flow, io.ldu_io.dcache.resp.bits.miss, io.stu_io.dcache.resp.bits.miss) // TODO: use trace with bank conflict? 1096 io.prefetch_train.bits.meta_prefetch := Mux(s2_ld_flow, io.ldu_io.dcache.resp.bits.meta_prefetch, false.B) 1097 io.prefetch_train.bits.meta_access := Mux(s2_ld_flow, io.ldu_io.dcache.resp.bits.meta_access, false.B) 1098 1099 io.prefetch_train_l1.valid := s2_valid && !s2_actually_mmio && s2_ld_flow 1100 io.prefetch_train_l1.bits.fromLsPipelineBundle(s2_in) 1101 io.prefetch_train_l1.bits.miss := io.ldu_io.dcache.resp.bits.miss 1102 io.prefetch_train_l1.bits.meta_prefetch := io.ldu_io.dcache.resp.bits.meta_prefetch 1103 io.prefetch_train_l1.bits.meta_access := io.ldu_io.dcache.resp.bits.meta_access 1104 if (env.FPGAPlatform){ 1105 io.ldu_io.dcache.s0_pc := DontCare 1106 io.ldu_io.dcache.s1_pc := DontCare 1107 io.ldu_io.dcache.s2_pc := DontCare 1108 }else{ 1109 io.ldu_io.dcache.s0_pc := s0_out.uop.pc 1110 io.ldu_io.dcache.s1_pc := s1_out.uop.pc 1111 io.ldu_io.dcache.s2_pc := s2_out.uop.pc 1112 } 1113 io.ldu_io.dcache.s2_kill := s2_pmp.ld || s2_actually_mmio || s2_kill 1114 io.stu_io.dcache.s2_kill := s2_pmp.st || s2_actually_mmio || s2_kill 1115 io.stu_io.dcache.s2_pc := s2_out.uop.pc 1116 1117 val s1_ld_left_fire = s1_valid && !s1_kill && s2_ready && s1_ld_flow 1118 val s2_ld_valid_dup = RegInit(0.U(6.W)) 1119 s2_ld_valid_dup := 0x0.U(6.W) 1120 when (s1_ld_left_fire && !s1_out.isHWPrefetch && s1_ld_flow) { s2_ld_valid_dup := 0x3f.U(6.W) } 1121 when (s1_kill || s1_out.isHWPrefetch || !s1_ld_flow) { s2_ld_valid_dup := 0x0.U(6.W) } 1122 assert(RegNext((s2_valid === s2_ld_valid_dup(0)) || RegNext(s1_out.isHWPrefetch) || RegNext(!s1_ld_flow))) 1123 1124 // Pipeline 1125 // -------------------------------------------------------------------------------- 1126 // stage 3 1127 // -------------------------------------------------------------------------------- 1128 // writeback and update load queue 1129 val s3_valid = RegNext(s2_valid && !s2_out.isHWPrefetch && !s2_out.uop.robIdx.needFlush(io.redirect)) 1130 val s3_in = RegEnable(s2_out, s2_fire) 1131 val s3_out = Wire(Valid(new MemExuOutput)) 1132 val s3_dcache_rep = RegEnable(s2_dcache_fast_rep && s2_troublem, false.B, s2_fire) 1133 val s3_ld_valid_dup = RegEnable(s2_ld_valid_dup, s2_fire) 1134 val s3_fast_rep = Wire(Bool()) 1135 val s3_ld_flow = RegNext(s2_ld_flow) 1136 val s3_troublem = RegNext(s2_troublem) 1137 val s3_kill = s3_in.uop.robIdx.needFlush(io.redirect) 1138 val s3_isvec = RegNext(s2_isvec) 1139 s3_ready := !s3_valid || s3_kill || sx_can_go 1140 1141 // s3 load fast replay 1142 io.ldu_io.fast_rep_out.valid := s3_valid && 1143 s3_fast_rep && 1144 !s3_in.uop.robIdx.needFlush(io.redirect) && 1145 s3_ld_flow && 1146 !s3_isvec 1147 io.ldu_io.fast_rep_out.bits := s3_in 1148 1149 io.ldu_io.lsq.ldin.valid := s3_valid && 1150 (!s3_fast_rep || !io.ldu_io.fast_rep_out.ready) && 1151 !s3_in.feedbacked && 1152 !s3_in.lateKill && 1153 s3_ld_flow 1154 io.ldu_io.lsq.ldin.bits := s3_in 1155 io.ldu_io.lsq.ldin.bits.miss := s3_in.miss 1156 1157 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1158 io.ldu_io.lsq.ldin.bits.data_wen_dup := s3_ld_valid_dup.asBools 1159 io.ldu_io.lsq.ldin.bits.replacementUpdated := io.ldu_io.dcache.resp.bits.replacementUpdated 1160 io.ldu_io.lsq.ldin.bits.missDbUpdated := RegNext(s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated) 1161 1162 val s3_dly_ld_err = 1163 if (EnableAccurateLoadError) { 1164 (s3_in.lateKill || io.ldu_io.dcache.resp.bits.error_delayed) && RegNext(io.csrCtrl.cache_error_enable) 1165 } else { 1166 WireInit(false.B) 1167 } 1168 io.ldu_io.s3_dly_ld_err := false.B // s3_dly_ld_err && s3_valid 1169 io.ldu_io.fast_rep_out.bits.delayedLoadError := s3_dly_ld_err 1170 io.ldu_io.lsq.ldin.bits.dcacheRequireReplay := s3_dcache_rep 1171 1172 val s3_vp_match_fail = RegNext(io.ldu_io.lsq.forward.matchInvalid || io.ldu_io.sbuffer.matchInvalid || io.ldu_io.ubuffer.matchInvalid) && s3_troublem 1173 val s3_ldld_rep_inst = 1174 io.ldu_io.lsq.ldld_nuke_query.resp.valid && 1175 io.ldu_io.lsq.ldld_nuke_query.resp.bits.rep_frm_fetch && 1176 RegNext(io.csrCtrl.ldld_vio_check_enable) 1177 1178 val s3_rep_info = WireInit(s3_in.rep_info) 1179 s3_rep_info.dcache_miss := s3_in.rep_info.dcache_miss && s3_troublem 1180 val s3_rep_frm_fetch = s3_vp_match_fail 1181 val s3_flushPipe = s3_ldld_rep_inst 1182 val s3_sel_rep_cause = PriorityEncoderOH(s3_rep_info.cause.asUInt) 1183 val s3_force_rep = s3_sel_rep_cause(LoadReplayCauses.C_TM) && 1184 !s3_in.uop.exceptionVec(loadAddrMisaligned) && 1185 s3_troublem 1186 1187 val s3_ld_exception = ExceptionNO.selectByFu(s3_in.uop.exceptionVec, LduCfg).asUInt.orR && s3_ld_flow 1188 val s3_st_exception = ExceptionNO.selectByFu(s3_in.uop.exceptionVec, StaCfg).asUInt.orR && !s3_ld_flow 1189 val s3_exception = s3_ld_exception || s3_st_exception 1190 when ((s3_ld_exception || s3_dly_ld_err || s3_rep_frm_fetch) && !s3_force_rep) { 1191 io.ldu_io.lsq.ldin.bits.rep_info.cause := 0.U.asTypeOf(s3_rep_info.cause.cloneType) 1192 } .otherwise { 1193 io.ldu_io.lsq.ldin.bits.rep_info.cause := VecInit(s3_sel_rep_cause.asBools) 1194 } 1195 1196 // Int flow, if hit, will be writebacked at s3 1197 s3_out.valid := s3_valid && 1198 (!s3_ld_flow && !s3_in.feedbacked || !io.ldu_io.lsq.ldin.bits.rep_info.need_rep) && !s3_in.mmio 1199 s3_out.bits.uop := s3_in.uop 1200 s3_out.bits.uop.exceptionVec(loadAccessFault) := (s3_dly_ld_err || s3_in.uop.exceptionVec(loadAccessFault)) && s3_ld_flow 1201 s3_out.bits.uop.replayInst := s3_rep_frm_fetch 1202 s3_out.bits.data := s3_in.data 1203 s3_out.bits.debug.isMMIO := s3_in.mmio 1204 s3_out.bits.debug.isNC := s3_in.nc 1205 s3_out.bits.debug.isPerfCnt := false.B 1206 s3_out.bits.debug.paddr := s3_in.paddr 1207 s3_out.bits.debug.vaddr := s3_in.vaddr 1208 1209 when (s3_force_rep) { 1210 s3_out.bits.uop.exceptionVec := 0.U.asTypeOf(s3_in.uop.exceptionVec.cloneType) 1211 } 1212 1213 io.ldu_io.rollback.valid := s3_valid && (s3_rep_frm_fetch || s3_flushPipe) && !s3_exception && s3_ld_flow 1214 io.ldu_io.rollback.bits := DontCare 1215 io.ldu_io.rollback.bits.isRVC := s3_out.bits.uop.preDecodeInfo.isRVC 1216 io.ldu_io.rollback.bits.robIdx := s3_out.bits.uop.robIdx 1217 io.ldu_io.rollback.bits.ftqIdx := s3_out.bits.uop.ftqPtr 1218 io.ldu_io.rollback.bits.ftqOffset := s3_out.bits.uop.ftqOffset 1219 io.ldu_io.rollback.bits.level := Mux(s3_rep_frm_fetch, RedirectLevel.flush, RedirectLevel.flushAfter) 1220 io.ldu_io.rollback.bits.cfiUpdate.target := s3_out.bits.uop.pc 1221 io.ldu_io.rollback.bits.debug_runahead_checkpoint_id := s3_out.bits.uop.debugInfo.runahead_checkpoint_id 1222 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1223 io.ldu_io.lsq.ldin.bits.uop := s3_out.bits.uop 1224 1225 val s3_revoke = s3_exception || io.ldu_io.lsq.ldin.bits.rep_info.need_rep 1226 io.ldu_io.lsq.ldld_nuke_query.revoke := s3_revoke 1227 io.ldu_io.lsq.stld_nuke_query.revoke := s3_revoke 1228 1229 // feedback slow 1230 s3_fast_rep := RegNext(s2_fast_rep) && 1231 !s3_in.feedbacked && 1232 !s3_in.lateKill && 1233 !s3_rep_frm_fetch && 1234 !s3_exception 1235 1236 val s3_fb_no_waiting = !s3_in.isLoadReplay && !(s3_fast_rep && io.ldu_io.fast_rep_out.ready) && !s3_in.feedbacked 1237 1238 // 1239 io.feedback_slow.valid := s3_valid && !s3_in.uop.robIdx.needFlush(io.redirect) && s3_fb_no_waiting && s3_ld_flow 1240 io.feedback_slow.bits.hit := !io.ldu_io.lsq.ldin.bits.rep_info.need_rep || io.ldu_io.lsq.ldin.ready 1241 io.feedback_slow.bits.flushState := s3_in.ptwBack 1242 io.feedback_slow.bits.robIdx := s3_in.uop.robIdx 1243 io.feedback_slow.bits.sourceType := RSFeedbackType.lrqFull 1244 io.feedback_slow.bits.dataInvalidSqIdx := DontCare 1245 1246 io.vec_stu_io.feedbackSlow.valid := RegNext(s2_vec_feedback.valid && !s2_out.uop.robIdx.needFlush(io.redirect)) 1247 io.vec_stu_io.feedbackSlow.bits := RegNext(s2_vec_feedback.bits) 1248 1249 io.ldu_io.ldCancel.ld2Cancel := s3_valid && s3_ld_flow && ( // is load 1250 io.ldu_io.lsq.ldin.bits.rep_info.need_rep || s3_in.mmio // exe fail or is mmio 1251 ) 1252 1253 // data from dcache hit 1254 val s3_ld_raw_data_frm_cache = Wire(new LoadDataFromDcacheBundle) 1255 s3_ld_raw_data_frm_cache.respDcacheData := io.ldu_io.dcache.resp.bits.data 1256 s3_ld_raw_data_frm_cache.forward_D := s2_fwd_frm_d_chan 1257 s3_ld_raw_data_frm_cache.forwardData_D := s2_fwd_data_frm_d_chan 1258 s3_ld_raw_data_frm_cache.forward_mshr := s2_fwd_frm_mshr 1259 s3_ld_raw_data_frm_cache.forwardData_mshr := s2_fwd_data_frm_mshr 1260 s3_ld_raw_data_frm_cache.forward_result_valid := s2_fwd_data_valid 1261 1262 s3_ld_raw_data_frm_cache.forwardMask := RegEnable(s2_fwd_mask, s2_valid) 1263 s3_ld_raw_data_frm_cache.forwardData := RegEnable(s2_fwd_data, s2_valid) 1264 s3_ld_raw_data_frm_cache.uop := RegEnable(s2_out.uop, s2_valid) 1265 s3_ld_raw_data_frm_cache.addrOffset := RegEnable(s2_out.paddr(3, 0), s2_valid) 1266 1267 val s3_merged_data_frm_tlD = RegEnable(s3_ld_raw_data_frm_cache.mergeTLData(), s2_valid) 1268 val s3_merged_data_frm_cache = s3_ld_raw_data_frm_cache.mergeLsqFwdData(s3_merged_data_frm_tlD) 1269 val s3_picked_data_frm_cache = LookupTree(s3_ld_raw_data_frm_cache.addrOffset, List( 1270 "b0000".U -> s3_merged_data_frm_cache(63, 0), 1271 "b0001".U -> s3_merged_data_frm_cache(63, 8), 1272 "b0010".U -> s3_merged_data_frm_cache(63, 16), 1273 "b0011".U -> s3_merged_data_frm_cache(63, 24), 1274 "b0100".U -> s3_merged_data_frm_cache(63, 32), 1275 "b0101".U -> s3_merged_data_frm_cache(63, 40), 1276 "b0110".U -> s3_merged_data_frm_cache(63, 48), 1277 "b0111".U -> s3_merged_data_frm_cache(63, 56), 1278 "b1000".U -> s3_merged_data_frm_cache(127, 64), 1279 "b1001".U -> s3_merged_data_frm_cache(127, 72), 1280 "b1010".U -> s3_merged_data_frm_cache(127, 80), 1281 "b1011".U -> s3_merged_data_frm_cache(127, 88), 1282 "b1100".U -> s3_merged_data_frm_cache(127, 96), 1283 "b1101".U -> s3_merged_data_frm_cache(127, 104), 1284 "b1110".U -> s3_merged_data_frm_cache(127, 112), 1285 "b1111".U -> s3_merged_data_frm_cache(127, 120) 1286 )) 1287 val s3_ld_data_frm_cache = rdataHelper(s3_ld_raw_data_frm_cache.uop, s3_picked_data_frm_cache) 1288 1289 // FIXME: add 1 cycle delay ? 1290 io.ldout.bits := s3_out.bits 1291 io.ldout.bits.data := s3_ld_data_frm_cache 1292 io.ldout.valid := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) && s3_ld_flow && !s3_isvec 1293 1294 // for uncache 1295 io.ldu_io.lsq.uncache.ready := true.B 1296 1297 // fast load to load forward 1298 if (EnableLoadToLoadForward) { 1299 io.ldu_io.l2l_fwd_out.valid := s3_out.valid && !s3_in.lateKill && s3_ld_flow 1300 io.ldu_io.l2l_fwd_out.data := s3_ld_data_frm_cache 1301 io.ldu_io.l2l_fwd_out.dly_ld_err := s3_dly_ld_err // ecc delayed error 1302 } else { 1303 io.ldu_io.l2l_fwd_out.valid := false.B 1304 io.ldu_io.l2l_fwd_out.data := DontCare 1305 io.ldu_io.l2l_fwd_out.dly_ld_err := DontCare 1306 } 1307 1308 // hybrid unit writeback to rob 1309 // delay params 1310 val SelectGroupSize = RollbackGroupSize 1311 val lgSelectGroupSize = log2Ceil(SelectGroupSize) 1312 val TotalSelectCycles = scala.math.ceil(log2Ceil(LoadQueueRAWSize).toFloat / lgSelectGroupSize).toInt + 1 1313 val TotalDelayCycles = TotalSelectCycles - 2 1314 1315 // writeback 1316 val sx_valid = Wire(Vec(TotalDelayCycles + 1, Bool())) 1317 val sx_ready = Wire(Vec(TotalDelayCycles + 1, Bool())) 1318 val sx_in = Wire(Vec(TotalDelayCycles + 1, new MemExuOutput)) 1319 1320 sx_can_go := sx_ready.head 1321 for (i <- 0 until TotalDelayCycles + 1) { 1322 if (i == 0) { 1323 sx_valid(i) := s3_valid && 1324 !s3_ld_flow && 1325 !s3_in.feedbacked && 1326 !s3_in.mmio 1327 sx_in(i) := s3_out.bits 1328 sx_ready(i) := !s3_valid(i) || sx_in(i).uop.robIdx.needFlush(io.redirect) || (if (TotalDelayCycles == 0) io.stout.ready else sx_ready(i+1)) 1329 } else { 1330 val cur_kill = sx_in(i).uop.robIdx.needFlush(io.redirect) 1331 val cur_can_go = (if (i == TotalDelayCycles) io.stout.ready else sx_ready(i+1)) 1332 val cur_fire = sx_valid(i) && !cur_kill && cur_can_go 1333 val prev_fire = sx_valid(i-1) && !sx_in(i-1).uop.robIdx.needFlush(io.redirect) && sx_ready(i) 1334 1335 sx_ready(i) := !sx_valid(i) || cur_kill || (if (i == TotalDelayCycles) io.stout.ready else sx_ready(i+1)) 1336 val sx_valid_can_go = prev_fire || cur_fire || cur_kill 1337 sx_valid(i) := RegEnable(Mux(prev_fire, true.B, false.B), sx_valid_can_go) 1338 sx_in(i) := RegEnable(sx_in(i-1), prev_fire) 1339 } 1340 } 1341 1342 val sx_last_valid = sx_valid.takeRight(1).head 1343 val sx_last_ready = sx_ready.takeRight(1).head 1344 val sx_last_in = sx_in.takeRight(1).head 1345 1346 sx_last_ready := !sx_last_valid || sx_last_in.uop.robIdx.needFlush(io.redirect) || io.stout.ready 1347 io.stout.valid := sx_last_valid && !sx_last_in.uop.robIdx.needFlush(io.redirect) && FuType.isStore(sx_last_in.uop.fuType) 1348 io.stout.bits := sx_last_in 1349 1350 // FIXME: please move this part to LoadQueueReplay 1351 io.ldu_io.debug_ls := DontCare 1352 io.stu_io.debug_ls := DontCare 1353 io.stu_io.debug_ls.s1_isTlbFirstMiss := io.tlb.resp.valid && io.tlb.resp.bits.miss && io.tlb.resp.bits.debug.isFirstIssue && !s1_in.isHWPrefetch && !s1_ld_flow 1354 io.stu_io.debug_ls.s1_robIdx := s1_in.uop.robIdx.value 1355 1356 // Topdown 1357 io.ldu_io.lsTopdownInfo.s1.robIdx := s1_in.uop.robIdx.value 1358 io.ldu_io.lsTopdownInfo.s1.vaddr_valid := s1_valid && s1_in.hasROBEntry 1359 io.ldu_io.lsTopdownInfo.s1.vaddr_bits := s1_vaddr 1360 io.ldu_io.lsTopdownInfo.s2.robIdx := s2_in.uop.robIdx.value 1361 io.ldu_io.lsTopdownInfo.s2.paddr_valid := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss 1362 io.ldu_io.lsTopdownInfo.s2.paddr_bits := s2_in.paddr 1363 io.ldu_io.lsTopdownInfo.s2.first_real_miss := io.ldu_io.dcache.resp.bits.real_miss 1364 io.ldu_io.lsTopdownInfo.s2.cache_miss_en := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated 1365 1366 // perf cnt 1367 XSPerfAccumulate("s0_in_valid", io.lsin.valid) 1368 XSPerfAccumulate("s0_in_block", io.lsin.valid && !io.lsin.fire) 1369 XSPerfAccumulate("s0_in_fire_first_issue", s0_valid && s0_isFirstIssue) 1370 XSPerfAccumulate("s0_lsq_fire_first_issue", io.ldu_io.replay.fire) 1371 XSPerfAccumulate("s0_ldu_fire_first_issue", io.lsin.fire && s0_isFirstIssue) 1372 XSPerfAccumulate("s0_fast_replay_issue", io.ldu_io.fast_rep_in.fire) 1373 XSPerfAccumulate("s0_stall_out", s0_valid && !s0_can_go) 1374 XSPerfAccumulate("s0_stall_ld_dcache", s0_valid && !io.ldu_io.dcache.req.ready) 1375 XSPerfAccumulate("s0_stall_st_dcache", s0_valid && !io.stu_io.dcache.req.ready) 1376 XSPerfAccumulate("s0_addr_spec_success", s0_fire && s0_vaddr(VAddrBits-1, 12) === io.lsin.bits.src(0)(VAddrBits-1, 12)) 1377 XSPerfAccumulate("s0_addr_spec_failed", s0_fire && s0_vaddr(VAddrBits-1, 12) =/= io.lsin.bits.src(0)(VAddrBits-1, 12)) 1378 XSPerfAccumulate("s0_addr_spec_success_once", s0_fire && s0_vaddr(VAddrBits-1, 12) === io.lsin.bits.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 1379 XSPerfAccumulate("s0_addr_spec_failed_once", s0_fire && s0_vaddr(VAddrBits-1, 12) =/= io.lsin.bits.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 1380 XSPerfAccumulate("s0_forward_tl_d_channel", s0_out.forward_tlDchannel) 1381 XSPerfAccumulate("s0_hardware_prefetch_fire", s0_fire && s0_hw_prf_select) 1382 XSPerfAccumulate("s0_software_prefetch_fire", s0_fire && s0_prf && s0_src_select_vec(int_iss_idx)) 1383 XSPerfAccumulate("s0_hardware_prefetch_blocked", io.ldu_io.prefetch_req.valid && !s0_hw_prf_select) 1384 XSPerfAccumulate("s0_hardware_prefetch_total", io.ldu_io.prefetch_req.valid) 1385 1386 XSPerfAccumulate("s1_in_valid", s1_valid) 1387 XSPerfAccumulate("s1_in_fire", s1_fire) 1388 XSPerfAccumulate("s1_in_fire_first_issue", s1_fire && s1_in.isFirstIssue) 1389 XSPerfAccumulate("s1_tlb_miss", s1_fire && s1_tlb_miss) 1390 XSPerfAccumulate("s1_tlb_miss_first_issue", s1_fire && s1_tlb_miss && s1_in.isFirstIssue) 1391 XSPerfAccumulate("s1_stall_out", s1_valid && !s1_can_go) 1392 XSPerfAccumulate("s1_late_kill", s1_valid && s1_fast_rep_kill) 1393 1394 XSPerfAccumulate("s2_in_valid", s2_valid) 1395 XSPerfAccumulate("s2_in_fire", s2_fire) 1396 XSPerfAccumulate("s2_in_fire_first_issue", s2_fire && s2_in.isFirstIssue) 1397 XSPerfAccumulate("s2_dcache_miss", s2_fire && io.ldu_io.dcache.resp.bits.miss) 1398 XSPerfAccumulate("s2_dcache_miss_first_issue", s2_fire && io.ldu_io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1399 XSPerfAccumulate("s2_dcache_real_miss_first_issue", s2_fire && io.ldu_io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1400 XSPerfAccumulate("s2_full_forward", s2_fire && s2_full_fwd) 1401 XSPerfAccumulate("s2_dcache_miss_full_forward", s2_fire && s2_dcache_miss) 1402 XSPerfAccumulate("s2_fwd_frm_d_can", s2_valid && s2_fwd_frm_d_chan) 1403 XSPerfAccumulate("s2_fwd_frm_d_chan_or_mshr", s2_valid && s2_fwd_frm_d_chan_or_mshr) 1404 XSPerfAccumulate("s2_stall_out", s2_fire && !s2_can_go) 1405 XSPerfAccumulate("s2_prefetch", s2_fire && s2_prf) 1406 XSPerfAccumulate("s2_prefetch_ignored", s2_fire && s2_prf && io.ldu_io.dcache.s2_mq_nack) // ignore prefetch for mshr full / miss req port conflict 1407 XSPerfAccumulate("s2_prefetch_miss", s2_fire && s2_prf && io.ldu_io.dcache.resp.bits.miss) // prefetch req miss in l1 1408 XSPerfAccumulate("s2_prefetch_hit", s2_fire && s2_prf && !io.ldu_io.dcache.resp.bits.miss) // prefetch req hit in l1 1409 XSPerfAccumulate("s2_prefetch_accept", s2_fire && s2_prf && io.ldu_io.dcache.resp.bits.miss && !io.ldu_io.dcache.s2_mq_nack) // prefetch a missed line in l1, and l1 accepted it 1410 XSPerfAccumulate("s2_forward_req", s2_fire && s2_in.forward_tlDchannel) 1411 XSPerfAccumulate("s2_successfully_forward_channel_D", s2_fire && s2_fwd_frm_d_chan && s2_fwd_data_valid) 1412 XSPerfAccumulate("s2_successfully_forward_mshr", s2_fire && s2_fwd_frm_mshr && s2_fwd_data_valid) 1413 1414 XSPerfAccumulate("load_to_load_forward", s1_try_ptr_chasing && !s1_ptr_chasing_canceled) 1415 XSPerfAccumulate("load_to_load_forward_try", s1_try_ptr_chasing) 1416 XSPerfAccumulate("load_to_load_forward_fail", s1_cancel_ptr_chasing) 1417 XSPerfAccumulate("load_to_load_forward_fail_cancelled", s1_cancel_ptr_chasing && s1_ptr_chasing_canceled) 1418 XSPerfAccumulate("load_to_load_forward_fail_wakeup_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && s1_not_fast_match) 1419 XSPerfAccumulate("load_to_load_forward_fail_op_not_ld", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && s1_fu_op_type_not_ld) 1420 XSPerfAccumulate("load_to_load_forward_fail_addr_align", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && s1_addr_misaligned) 1421 XSPerfAccumulate("load_to_load_forward_fail_set_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && !s1_addr_misaligned && s1_addr_mismatch) 1422 1423 // bug lyq: some signals in perfEvents are no longer suitable for the current MemBlock design 1424 // hardware performance counter 1425 val perfEvents = Seq( 1426 ("load_s0_in_fire ", s0_fire ), 1427 ("load_to_load_forward ", s1_fire && s1_try_ptr_chasing && !s1_ptr_chasing_canceled ), 1428 ("stall_dcache ", s0_valid && s0_can_go && !io.ldu_io.dcache.req.ready ), 1429 ("load_s1_in_fire ", s0_fire ), 1430 ("load_s1_tlb_miss ", s1_fire && io.tlb.resp.bits.miss ), 1431 ("load_s2_in_fire ", s1_fire ), 1432 ("load_s2_dcache_miss ", s2_fire && io.ldu_io.dcache.resp.bits.miss ), 1433 ) 1434 generatePerfEvent() 1435} 1436