1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.Bundles.{DynInst, MemExuInput, MemExuOutput} 27import xiangshan.backend.fu.PMPRespBundle 28import xiangshan.backend.fu.FuConfig._ 29import xiangshan.backend.ctrlblock.{DebugLsInfoBundle, LsTopdownInfo} 30import xiangshan.backend.rob.RobPtr 31import xiangshan.backend.fu._ 32import xiangshan.backend.fu.util.SdtrigExt 33import xiangshan.cache._ 34import xiangshan.cache.wpu.ReplayCarry 35import xiangshan.cache.mmu.{TlbCmd, TlbHintReq, TlbReq, TlbRequestIO, TlbResp} 36import xiangshan.mem.mdp._ 37 38class HybridUnit(implicit p: Parameters) extends XSModule 39 with HasLoadHelper 40 with HasPerfEvents 41 with HasDCacheParameters 42 with HasCircularQueuePtrHelper 43 with HasVLSUParameters 44 with SdtrigExt 45{ 46 val io = IO(new Bundle() { 47 // control 48 val redirect = Flipped(ValidIO(new Redirect)) 49 val csrCtrl = Flipped(new CustomCSRCtrlIO) 50 51 // flow in 52 val lsin = Flipped(Decoupled(new MemExuInput)) 53 54 // flow out 55 val ldout = DecoupledIO(new MemExuOutput) 56 val stout = DecoupledIO(new MemExuOutput) 57 58 val ldu_io = new Bundle() { 59 // dcache 60 val dcache = new DCacheLoadIO 61 62 // data path 63 val sbuffer = new LoadForwardQueryIO 64 val vec_forward = new LoadForwardQueryIO 65 val lsq = new LoadToLsqIO 66 val tl_d_channel = Input(new DcacheToLduForwardIO) 67 val forward_mshr = Flipped(new LduToMissqueueForwardIO) 68 val refill = Flipped(ValidIO(new Refill)) 69 val tlb_hint = Flipped(new TlbHintReq) 70 val l2_hint = Input(Valid(new L2ToL1Hint)) 71 72 // fast wakeup 73 val fast_uop = ValidIO(new DynInst) // early wakeup signal generated in load_s1, send to RS in load_s2 74 75 // trigger 76 val trigger = Vec(TriggerNum, new LoadUnitTriggerIO) 77 78 // load to load fast path 79 val l2l_fwd_in = Input(new LoadToLoadIO) 80 val l2l_fwd_out = Output(new LoadToLoadIO) 81 82 val ld_fast_match = Input(Bool()) 83 val ld_fast_fuOpType = Input(UInt()) 84 val ld_fast_imm = Input(UInt(12.W)) 85 86 // hardware prefetch to l1 cache req 87 val prefetch_req = Flipped(ValidIO(new L1PrefetchReq)) 88 89 // iq cancel 90 val ldCancel = Output(new LoadCancelIO()) // use to cancel the uops waked by this load, and cancel load 91 92 // load ecc error 93 val s3_dly_ld_err = Output(Bool()) // Note that io.s3_dly_ld_err and io.lsq.s3_dly_ld_err is different 94 95 // schedule error query 96 val stld_nuke_query = Flipped(Vec(StorePipelineWidth, Valid(new StoreNukeQueryIO))) 97 98 // queue-based replay 99 val replay = Flipped(Decoupled(new LsPipelineBundle)) 100 val lq_rep_full = Input(Bool()) 101 102 // misc 103 val s2_ptr_chasing = Output(Bool()) // provide right pc for hw prefetch 104 105 // Load fast replay path 106 val fast_rep_in = Flipped(Decoupled(new LqWriteBundle)) 107 val fast_rep_out = Decoupled(new LqWriteBundle) 108 109 // Load RAR rollback 110 val rollback = Valid(new Redirect) 111 112 // perf 113 val debug_ls = Output(new DebugLsInfoBundle) 114 val lsTopdownInfo = Output(new LsTopdownInfo) 115 } 116 117 val stu_io = new Bundle() { 118 val dcache = new DCacheStoreIO 119 val prefetch_req = Flipped(DecoupledIO(new StorePrefetchReq)) 120 val issue = Valid(new MemExuInput) 121 val lsq = ValidIO(new LsPipelineBundle) 122 val lsq_replenish = Output(new LsPipelineBundle()) 123 val stld_nuke_query = Valid(new StoreNukeQueryIO) 124 val st_mask_out = Valid(new StoreMaskBundle) 125 val debug_ls = Output(new DebugLsInfoBundle) 126 } 127 128 val vec_stu_io = new Bundle() { 129 val in = Flipped(DecoupledIO(new VecStorePipeBundle())) 130 val isFirstIssue = Input(Bool()) 131 val lsq = ValidIO(new LsPipelineBundle()) 132 val feedbackSlow = ValidIO(new VSFQFeedback) 133 } 134 135 // prefetch 136 val prefetch_train = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to sms 137 val prefetch_train_l1 = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to stream & stride 138 val canAcceptLowConfPrefetch = Output(Bool()) 139 val canAcceptHighConfPrefetch = Output(Bool()) 140 val correctMissTrain = Input(Bool()) 141 142 // data path 143 val tlb = new TlbRequestIO(2) 144 val pmp = Flipped(new PMPRespBundle()) // arrive same to tlb now 145 146 // rs feedback 147 val feedback_fast = ValidIO(new RSFeedback) // stage 2 148 val feedback_slow = ValidIO(new RSFeedback) // stage 3 149 }) 150 151 val StorePrefetchL1Enabled = EnableStorePrefetchAtCommit || EnableStorePrefetchAtIssue || EnableStorePrefetchSPB 152 val s1_ready, s2_ready, s3_ready, sx_can_go = WireInit(false.B) 153 154 // Pipeline 155 // -------------------------------------------------------------------------------- 156 // stage 0 157 // -------------------------------------------------------------------------------- 158 // generate addr, use addr to query DCache and DTLB 159 val s0_valid = Wire(Bool()) 160 val s0_dcache_ready = Wire(Bool()) 161 val s0_kill = Wire(Bool()) 162 val s0_vaddr = Wire(UInt(VAddrBits.W)) 163 val s0_mask = Wire(UInt((VLEN/8).W)) 164 val s0_uop = Wire(new DynInst) 165 val s0_has_rob_entry = Wire(Bool()) 166 val s0_rsIdx = Wire(UInt(log2Up(MemIQSizeMax).W)) 167 val s0_mshrid = Wire(UInt()) 168 val s0_try_l2l = Wire(Bool()) 169 val s0_rep_carry = Wire(new ReplayCarry(nWays)) 170 val s0_isFirstIssue = Wire(Bool()) 171 val s0_fast_rep = Wire(Bool()) 172 val s0_ld_rep = Wire(Bool()) 173 val s0_l2l_fwd = Wire(Bool()) 174 val s0_sched_idx = Wire(UInt()) 175 val s0_deqPortIdx = Wire(UInt(log2Ceil(LoadPipelineWidth).W)) 176 val s0_can_go = s1_ready 177 val s0_fire = s0_valid && s0_dcache_ready && s0_can_go 178 val s0_out = Wire(new LqWriteBundle) 179 // vector 180 val s0_isvec = WireInit(false.B) 181 val s0_exp = WireInit(true.B) 182 val s0_flowPtr = WireInit(0.U.asTypeOf(new VsFlowPtr)) 183 val s0_isLastElem = WireInit(false.B) 184 185 // load flow select/gen 186 // src0: super load replayed by LSQ (cache miss replay) (io.ldu_io.replay) 187 // src1: fast load replay (io.ldu_io.fast_rep_in) 188 // src2: load replayed by LSQ (io.ldu_io.replay) 189 // src3: hardware prefetch from prefetchor (high confidence) (io.prefetch) 190 // src4: int read / software prefetch first issue from RS (io.in) 191 // src5: vec read first issue from RS (TODO) 192 // src6: load try pointchaising when no issued or replayed load (io.fastpath) 193 // src7: hardware prefetch from prefetchor (high confidence) (io.prefetch) 194 // priority: high to low 195 val s0_ld_flow = FuType.isLoad(s0_uop.fuType) || FuType.isVLoad(s0_uop.fuType) 196 val s0_rep_stall = io.lsin.valid && isAfter(io.ldu_io.replay.bits.uop.robIdx, io.lsin.bits.uop.robIdx) 197 val s0_super_ld_rep_valid = io.ldu_io.replay.valid && io.ldu_io.replay.bits.forward_tlDchannel 198 val s0_ld_fast_rep_valid = io.ldu_io.fast_rep_in.valid 199 val s0_ld_rep_valid = io.ldu_io.replay.valid && !io.ldu_io.replay.bits.forward_tlDchannel && !s0_rep_stall 200 val s0_high_conf_prf_valid = io.ldu_io.prefetch_req.valid && io.ldu_io.prefetch_req.bits.confidence > 0.U 201 val s0_int_iss_valid = io.lsin.valid // int flow first issue or software prefetch 202 val s0_vec_iss_valid = io.vec_stu_io.in.valid 203 val s0_l2l_fwd_valid = io.ldu_io.l2l_fwd_in.valid && io.ldu_io.ld_fast_match 204 val s0_low_conf_prf_valid = io.ldu_io.prefetch_req.valid && io.ldu_io.prefetch_req.bits.confidence === 0.U 205 dontTouch(s0_super_ld_rep_valid) 206 dontTouch(s0_ld_fast_rep_valid) 207 dontTouch(s0_ld_rep_valid) 208 dontTouch(s0_high_conf_prf_valid) 209 dontTouch(s0_int_iss_valid) 210 dontTouch(s0_vec_iss_valid) 211 dontTouch(s0_l2l_fwd_valid) 212 dontTouch(s0_low_conf_prf_valid) 213 214 // load flow source ready 215 val s0_super_ld_rep_ready = WireInit(true.B) 216 val s0_ld_fast_rep_ready = !s0_super_ld_rep_valid 217 val s0_ld_rep_ready = !s0_super_ld_rep_valid && 218 !s0_ld_fast_rep_valid 219 val s0_high_conf_prf_ready = !s0_super_ld_rep_valid && 220 !s0_ld_fast_rep_valid && 221 !s0_ld_rep_valid 222 223 val s0_int_iss_ready = !s0_super_ld_rep_valid && 224 !s0_ld_fast_rep_valid && 225 !s0_ld_rep_valid && 226 !s0_high_conf_prf_valid 227 228 val s0_vec_iss_ready = !s0_super_ld_rep_valid && 229 !s0_ld_fast_rep_valid && 230 !s0_ld_rep_valid && 231 !s0_high_conf_prf_valid && 232 !s0_int_iss_valid 233 234 val s0_l2l_fwd_ready = !s0_super_ld_rep_valid && 235 !s0_ld_fast_rep_valid && 236 !s0_ld_rep_valid && 237 !s0_high_conf_prf_valid && 238 !s0_int_iss_valid && 239 !s0_vec_iss_valid 240 241 val s0_low_conf_prf_ready = !s0_super_ld_rep_valid && 242 !s0_ld_fast_rep_valid && 243 !s0_ld_rep_valid && 244 !s0_high_conf_prf_valid && 245 !s0_int_iss_valid && 246 !s0_vec_iss_valid && 247 !s0_l2l_fwd_valid 248 dontTouch(s0_super_ld_rep_ready) 249 dontTouch(s0_ld_fast_rep_ready) 250 dontTouch(s0_ld_rep_ready) 251 dontTouch(s0_high_conf_prf_ready) 252 dontTouch(s0_int_iss_ready) 253 dontTouch(s0_vec_iss_ready) 254 dontTouch(s0_l2l_fwd_ready) 255 dontTouch(s0_low_conf_prf_ready) 256 257 // load flow source select (OH) 258 val s0_super_ld_rep_select = s0_super_ld_rep_valid && s0_super_ld_rep_ready 259 val s0_ld_fast_rep_select = s0_ld_fast_rep_valid && s0_ld_fast_rep_ready 260 val s0_ld_rep_select = s0_ld_rep_valid && s0_ld_rep_ready 261 val s0_hw_prf_select = s0_high_conf_prf_ready && s0_high_conf_prf_valid || 262 s0_low_conf_prf_ready && s0_low_conf_prf_valid 263 val s0_int_iss_select = s0_int_iss_ready && s0_int_iss_valid 264 val s0_vec_iss_select = s0_vec_iss_ready && s0_vec_iss_valid 265 val s0_l2l_fwd_select = s0_l2l_fwd_ready && s0_l2l_fwd_valid 266 dontTouch(s0_super_ld_rep_select) 267 dontTouch(s0_ld_fast_rep_select) 268 dontTouch(s0_ld_rep_select) 269 dontTouch(s0_hw_prf_select) 270 dontTouch(s0_int_iss_select) 271 dontTouch(s0_vec_iss_select) 272 dontTouch(s0_l2l_fwd_select) 273 274 s0_valid := (s0_super_ld_rep_valid || 275 s0_ld_fast_rep_valid || 276 s0_ld_rep_valid || 277 s0_high_conf_prf_valid || 278 s0_int_iss_valid || 279 s0_vec_iss_valid || 280 s0_l2l_fwd_valid || 281 s0_low_conf_prf_valid) && !s0_kill 282 283 // which is S0's out is ready and dcache is ready 284 val s0_try_ptr_chasing = s0_l2l_fwd_select 285 val s0_do_try_ptr_chasing = s0_try_ptr_chasing && s0_can_go && io.ldu_io.dcache.req.ready 286 val s0_ptr_chasing_vaddr = io.ldu_io.l2l_fwd_in.data(5, 0) +& io.ldu_io.ld_fast_imm(5, 0) 287 val s0_ptr_chasing_canceled = WireInit(false.B) 288 s0_kill := s0_ptr_chasing_canceled || (s0_out.uop.robIdx.needFlush(io.redirect) && !s0_try_ptr_chasing) 289 290 // prefetch related ctrl signal 291 val s0_prf = Wire(Bool()) 292 val s0_prf_rd = Wire(Bool()) 293 val s0_prf_wr = Wire(Bool()) 294 val s0_hw_prf = s0_hw_prf_select 295 296 io.canAcceptLowConfPrefetch := s0_low_conf_prf_ready 297 io.canAcceptHighConfPrefetch := s0_high_conf_prf_ready 298 299 if (StorePrefetchL1Enabled) { 300 s0_dcache_ready := Mux(s0_ld_flow, io.ldu_io.dcache.req.ready, io.stu_io.dcache.req.ready) 301 } else { 302 s0_dcache_ready := Mux(s0_ld_flow, io.ldu_io.dcache.req.ready, true.B) 303 } 304 305 // query DTLB 306 io.tlb.req.valid := s0_valid && s0_dcache_ready 307 io.tlb.req.bits.cmd := Mux(s0_prf, 308 Mux(s0_prf_wr, TlbCmd.write, TlbCmd.read), 309 Mux(s0_ld_flow, TlbCmd.read, TlbCmd.write) 310 ) 311 io.tlb.req.bits.vaddr := Mux(s0_hw_prf_select, io.ldu_io.prefetch_req.bits.paddr, s0_vaddr) 312 io.tlb.req.bits.size := Mux(s0_isvec, io.vec_stu_io.in.bits.alignedType, LSUOpType.size(s0_uop.fuOpType)) 313 io.tlb.req.bits.kill := s0_kill 314 io.tlb.req.bits.memidx.is_ld := s0_ld_flow 315 io.tlb.req.bits.memidx.is_st := !s0_ld_flow 316 io.tlb.req.bits.memidx.idx := s0_uop.lqIdx.value 317 io.tlb.req.bits.debug.robIdx := s0_uop.robIdx 318 io.tlb.req.bits.no_translate := s0_hw_prf_select // hw b.reqetch addr does not need to be translated 319 io.tlb.req.bits.debug.pc := s0_uop.pc 320 io.tlb.req.bits.debug.isFirstIssue := s0_isFirstIssue 321 322 // query DCache 323 // for load 324 io.ldu_io.dcache.req.valid := s0_valid && s0_dcache_ready && s0_ld_flow 325 io.ldu_io.dcache.req.bits.cmd := Mux(s0_prf_rd, MemoryOpConstants.M_PFR, 326 Mux(s0_prf_wr, MemoryOpConstants.M_PFW, MemoryOpConstants.M_XRD)) 327 io.ldu_io.dcache.req.bits.vaddr := s0_vaddr 328 io.ldu_io.dcache.req.bits.mask := s0_mask 329 io.ldu_io.dcache.req.bits.data := DontCare 330 io.ldu_io.dcache.req.bits.isFirstIssue := s0_isFirstIssue 331 io.ldu_io.dcache.req.bits.instrtype := Mux(s0_prf, DCACHE_PREFETCH_SOURCE.U, LOAD_SOURCE.U) 332 io.ldu_io.dcache.req.bits.debug_robIdx := s0_uop.robIdx.value 333 io.ldu_io.dcache.req.bits.replayCarry := s0_rep_carry 334 io.ldu_io.dcache.req.bits.id := DontCare // TODO: update cache meta 335 io.ldu_io.dcache.pf_source := Mux(s0_hw_prf_select, io.ldu_io.prefetch_req.bits.pf_source.value, L1_HW_PREFETCH_NULL) 336 337 // for store 338 io.stu_io.dcache.req.valid := s0_valid && s0_dcache_ready && !s0_ld_flow && !s0_prf 339 io.stu_io.dcache.req.bits.cmd := MemoryOpConstants.M_PFW 340 io.stu_io.dcache.req.bits.vaddr := s0_vaddr 341 io.stu_io.dcache.req.bits.instrtype := Mux(s0_prf, DCACHE_PREFETCH_SOURCE.U, STORE_SOURCE.U) 342 343 // load flow priority mux 344 def fromNullSource() = { 345 s0_vaddr := 0.U 346 s0_mask := 0.U 347 s0_uop := 0.U.asTypeOf(new DynInst) 348 s0_try_l2l := false.B 349 s0_has_rob_entry := false.B 350 s0_rsIdx := 0.U 351 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 352 s0_mshrid := 0.U 353 s0_isFirstIssue := false.B 354 s0_fast_rep := false.B 355 s0_ld_rep := false.B 356 s0_l2l_fwd := false.B 357 s0_prf := false.B 358 s0_prf_rd := false.B 359 s0_prf_wr := false.B 360 s0_sched_idx := 0.U 361 s0_deqPortIdx := 0.U 362 } 363 364 def fromFastReplaySource(src: LqWriteBundle) = { 365 s0_vaddr := src.vaddr 366 s0_mask := src.mask 367 s0_uop := src.uop 368 s0_try_l2l := false.B 369 s0_has_rob_entry := src.hasROBEntry 370 s0_rep_carry := src.rep_info.rep_carry 371 s0_mshrid := src.rep_info.mshr_id 372 s0_rsIdx := src.rsIdx 373 s0_isFirstIssue := false.B 374 s0_fast_rep := true.B 375 s0_ld_rep := src.isLoadReplay 376 s0_l2l_fwd := false.B 377 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 378 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 379 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 380 s0_sched_idx := src.schedIndex 381 s0_deqPortIdx := src.deqPortIdx 382 } 383 384 def fromNormalReplaySource(src: LsPipelineBundle) = { 385 s0_vaddr := src.vaddr 386 s0_mask := genVWmask(src.vaddr, src.uop.fuOpType(1, 0)) 387 s0_uop := src.uop 388 s0_try_l2l := false.B 389 s0_has_rob_entry := true.B 390 s0_rsIdx := src.rsIdx 391 s0_rep_carry := src.replayCarry 392 s0_mshrid := src.mshrid 393 s0_isFirstIssue := false.B 394 s0_fast_rep := false.B 395 s0_ld_rep := true.B 396 s0_l2l_fwd := false.B 397 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 398 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 399 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 400 s0_sched_idx := src.schedIndex 401 s0_deqPortIdx := src.deqPortIdx 402 } 403 404 def fromPrefetchSource(src: L1PrefetchReq) = { 405 s0_vaddr := src.getVaddr() 406 s0_mask := 0.U 407 s0_uop := DontCare 408 s0_try_l2l := false.B 409 s0_has_rob_entry := false.B 410 s0_rsIdx := 0.U 411 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 412 s0_mshrid := 0.U 413 s0_isFirstIssue := false.B 414 s0_fast_rep := false.B 415 s0_ld_rep := false.B 416 s0_l2l_fwd := false.B 417 s0_prf := true.B 418 s0_prf_rd := !src.is_store 419 s0_prf_wr := src.is_store 420 s0_sched_idx := 0.U 421 s0_deqPortIdx := 0.U 422 } 423 424 def fromIntIssueSource(src: MemExuInput) = { 425 s0_vaddr := src.src(0) + SignExt(src.uop.imm(11, 0), VAddrBits) 426 s0_mask := genVWmask(s0_vaddr, src.uop.fuOpType(1,0)) 427 s0_uop := src.uop 428 s0_try_l2l := false.B 429 s0_has_rob_entry := true.B 430 s0_rsIdx := src.iqIdx 431 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 432 s0_mshrid := 0.U 433 s0_isFirstIssue := true.B 434 s0_fast_rep := false.B 435 s0_ld_rep := false.B 436 s0_l2l_fwd := false.B 437 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 438 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 439 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 440 s0_sched_idx := 0.U 441 s0_deqPortIdx := src.deqPortIdx 442 } 443 444 def fromVecIssueSource(src: VecStorePipeBundle) = { 445 // For now, vector port handles only vector store flows 446 s0_vaddr := src.vaddr 447 s0_mask := src.mask 448 s0_uop := src.uop 449 s0_try_l2l := false.B 450 s0_has_rob_entry := true.B 451 s0_rsIdx := 0.U 452 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 453 s0_mshrid := 0.U 454 s0_isFirstIssue := src.isFirstIssue 455 s0_fast_rep := false.B 456 s0_ld_rep := false.B 457 s0_l2l_fwd := false.B 458 s0_prf := false.B 459 s0_prf_rd := false.B 460 s0_prf_wr := false.B 461 s0_sched_idx := 0.U 462 463 s0_isvec := true.B 464 s0_exp := io.vec_stu_io.in.bits.exp 465 s0_flowPtr := io.vec_stu_io.in.bits.flowPtr 466 s0_isLastElem := io.vec_stu_io.in.bits.isLastElem 467 s0_deqPortIdx := 0.U 468 } 469 470 def fromLoadToLoadSource(src: LoadToLoadIO) = { 471 s0_vaddr := Cat(src.data(XLEN-1, 6), s0_ptr_chasing_vaddr(5,0)) 472 s0_mask := genVWmask(s0_vaddr, io.ldu_io.ld_fast_fuOpType(1, 0)) 473 // When there's no valid instruction from RS and LSQ, we try the load-to-load forwarding. 474 // Assume the pointer chasing is always ld. 475 s0_uop.fuOpType := io.ldu_io.ld_fast_fuOpType 476 s0_try_l2l := true.B 477 // we dont care s0_isFirstIssue and s0_rsIdx and s0_sqIdx and s0_deqPortIdx in S0 when trying pointchasing 478 // because these signals will be updated in S1 479 s0_has_rob_entry := false.B 480 s0_rsIdx := 0.U 481 s0_mshrid := 0.U 482 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 483 s0_isFirstIssue := true.B 484 s0_fast_rep := false.B 485 s0_ld_rep := false.B 486 s0_l2l_fwd := true.B 487 s0_prf := false.B 488 s0_prf_rd := false.B 489 s0_prf_wr := false.B 490 s0_sched_idx := 0.U 491 s0_deqPortIdx := 0.U 492 } 493 494 // set default 495 s0_uop := DontCare 496 when (s0_super_ld_rep_select) { fromNormalReplaySource(io.ldu_io.replay.bits) } 497 .elsewhen (s0_ld_fast_rep_select) { fromFastReplaySource(io.ldu_io.fast_rep_in.bits) } 498 .elsewhen (s0_ld_rep_select) { fromNormalReplaySource(io.ldu_io.replay.bits) } 499 .elsewhen (s0_hw_prf_select) { fromPrefetchSource(io.ldu_io.prefetch_req.bits) } 500 .elsewhen (s0_int_iss_select) { fromIntIssueSource(io.lsin.bits) } 501 .elsewhen (s0_vec_iss_select) { fromVecIssueSource(io.vec_stu_io.in.bits) } 502 .otherwise { 503 if (EnableLoadToLoadForward) { 504 fromLoadToLoadSource(io.ldu_io.l2l_fwd_in) 505 } else { 506 fromNullSource() 507 } 508 } 509 510 // address align check 511 val s0_addr_aligned = LookupTree(Mux(s0_isvec, io.vec_stu_io.in.bits.alignedType, s0_uop.fuOpType(1, 0)), List( 512 "b00".U -> true.B, //b 513 "b01".U -> (s0_vaddr(0) === 0.U), //h 514 "b10".U -> (s0_vaddr(1, 0) === 0.U), //w 515 "b11".U -> (s0_vaddr(2, 0) === 0.U) //d 516 )) 517 518 // accept load flow if dcache ready (tlb is always ready) 519 // TODO: prefetch need writeback to loadQueueFlag 520 s0_out := DontCare 521 s0_out.rsIdx := s0_rsIdx 522 s0_out.vaddr := s0_vaddr 523 s0_out.mask := s0_mask 524 s0_out.uop := s0_uop 525 s0_out.isFirstIssue := s0_isFirstIssue 526 s0_out.hasROBEntry := s0_has_rob_entry 527 s0_out.isPrefetch := s0_prf 528 s0_out.isHWPrefetch := s0_hw_prf 529 s0_out.isFastReplay := s0_fast_rep 530 s0_out.isLoadReplay := s0_ld_rep 531 s0_out.isFastPath := s0_l2l_fwd 532 s0_out.mshrid := s0_mshrid 533 s0_out.isvec := s0_isvec 534 s0_out.isLastElem := s0_isLastElem 535 s0_out.exp := s0_exp 536 s0_out.sflowPtr := s0_flowPtr 537 s0_out.uop.exceptionVec(loadAddrMisaligned) := !s0_addr_aligned && s0_ld_flow 538 s0_out.uop.exceptionVec(storeAddrMisaligned) := !s0_addr_aligned && !s0_ld_flow 539 s0_out.forward_tlDchannel := s0_super_ld_rep_select 540 when(io.tlb.req.valid && s0_isFirstIssue) { 541 s0_out.uop.debugInfo.tlbFirstReqTime := GTimer() 542 }.otherwise{ 543 s0_out.uop.debugInfo.tlbFirstReqTime := s0_uop.debugInfo.tlbFirstReqTime 544 } 545 s0_out.schedIndex := s0_sched_idx 546 s0_out.deqPortIdx := s0_deqPortIdx 547 548 // load fast replay 549 io.ldu_io.fast_rep_in.ready := (s0_can_go && io.ldu_io.dcache.req.ready && s0_ld_fast_rep_ready) 550 551 // load flow source ready 552 // cache missed load has highest priority 553 // always accept cache missed load flow from load replay queue 554 io.ldu_io.replay.ready := (s0_can_go && io.ldu_io.dcache.req.ready && (s0_ld_rep_ready && !s0_rep_stall || s0_super_ld_rep_select)) 555 556 // accept load flow from rs when: 557 // 1) there is no lsq-replayed load 558 // 2) there is no fast replayed load 559 // 3) there is no high confidence prefetch request 560 io.lsin.ready := (s0_can_go && 561 Mux(FuType.isLoad(io.lsin.bits.uop.fuType), io.ldu_io.dcache.req.ready, 562 (if (StorePrefetchL1Enabled) io.stu_io.dcache.req.ready else true.B)) && s0_int_iss_ready) 563 io.vec_stu_io.in.ready := s0_can_go && io.ldu_io.dcache.req.ready && s0_vec_iss_ready 564 565 566 // for hw prefetch load flow feedback, to be added later 567 // io.prefetch_in.ready := s0_hw_prf_select 568 569 // dcache replacement extra info 570 // TODO: should prefetch load update replacement? 571 io.ldu_io.dcache.replacementUpdated := Mux(s0_ld_rep_select || s0_super_ld_rep_select, io.ldu_io.replay.bits.replacementUpdated, false.B) 572 573 io.stu_io.prefetch_req.ready := s1_ready && io.stu_io.dcache.req.ready && !io.lsin.valid 574 575 // load debug 576 XSDebug(io.ldu_io.dcache.req.fire && s0_ld_flow, 577 p"[DCACHE LOAD REQ] pc ${Hexadecimal(s0_uop.pc)}, vaddr ${Hexadecimal(s0_vaddr)}\n" 578 ) 579 XSDebug(s0_valid && s0_ld_flow, 580 p"S0: pc ${Hexadecimal(s0_out.uop.pc)}, lqIdx ${Hexadecimal(s0_out.uop.lqIdx.asUInt)}, " + 581 p"vaddr ${Hexadecimal(s0_out.vaddr)}, mask ${Hexadecimal(s0_out.mask)}\n") 582 583 // store debug 584 XSDebug(io.stu_io.dcache.req.fire && !s0_ld_flow, 585 p"[DCACHE STORE REQ] pc ${Hexadecimal(s0_uop.pc)}, vaddr ${Hexadecimal(s0_vaddr)}\n" 586 ) 587 XSDebug(s0_valid && !s0_ld_flow, 588 p"S0: pc ${Hexadecimal(s0_out.uop.pc)}, sqIdx ${Hexadecimal(s0_out.uop.sqIdx.asUInt)}, " + 589 p"vaddr ${Hexadecimal(s0_out.vaddr)}, mask ${Hexadecimal(s0_out.mask)}\n") 590 591 592 // Pipeline 593 // -------------------------------------------------------------------------------- 594 // stage 1 595 // -------------------------------------------------------------------------------- 596 // TLB resp (send paddr to dcache) 597 val s1_valid = RegInit(false.B) 598 val s1_in = Wire(new LqWriteBundle) 599 val s1_out = Wire(new LqWriteBundle) 600 val s1_kill = Wire(Bool()) 601 val s1_can_go = s2_ready 602 val s1_fire = s1_valid && !s1_kill && s1_can_go 603 val s1_ld_flow = RegNext(s0_ld_flow) 604 val s1_isvec = RegEnable(s0_out.isvec, false.B, s0_fire) 605 val s1_isLastElem = RegEnable(s0_out.isLastElem, false.B, s0_fire) 606 607 s1_ready := !s1_valid || s1_kill || s2_ready 608 when (s0_fire) { s1_valid := true.B } 609 .elsewhen (s1_fire) { s1_valid := false.B } 610 .elsewhen (s1_kill) { s1_valid := false.B } 611 s1_in := RegEnable(s0_out, s0_fire) 612 613 val s1_fast_rep_dly_err = RegNext(io.ldu_io.fast_rep_in.bits.delayedLoadError) 614 val s1_fast_rep_kill = s1_fast_rep_dly_err && s1_in.isFastReplay 615 val s1_l2l_fwd_dly_err = RegNext(io.ldu_io.l2l_fwd_in.dly_ld_err) 616 val s1_l2l_fwd_kill = s1_l2l_fwd_dly_err && s1_in.isFastPath 617 val s1_late_kill = s1_fast_rep_kill || s1_l2l_fwd_kill 618 val s1_vaddr_hi = Wire(UInt()) 619 val s1_vaddr_lo = Wire(UInt()) 620 val s1_vaddr = Wire(UInt()) 621 val s1_paddr_dup_lsu = Wire(UInt()) 622 val s1_paddr_dup_dcache = Wire(UInt()) 623 val s1_ld_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, LduCfg).asUInt.orR // af & pf exception were modified below. 624 val s1_st_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, StaCfg).asUInt.orR // af & pf exception were modified below. 625 val s1_exception = (s1_ld_flow && s1_ld_exception) || (!s1_ld_flow && s1_st_exception) 626 val s1_tlb_miss = io.tlb.resp.bits.miss 627 val s1_prf = s1_in.isPrefetch 628 val s1_hw_prf = s1_in.isHWPrefetch 629 val s1_sw_prf = s1_prf && !s1_hw_prf 630 val s1_tlb_memidx = io.tlb.resp.bits.memidx 631 632 // mmio cbo decoder 633 val s1_mmio_cbo = (s1_in.uop.fuOpType === LSUOpType.cbo_clean || 634 s1_in.uop.fuOpType === LSUOpType.cbo_flush || 635 s1_in.uop.fuOpType === LSUOpType.cbo_inval) && !s1_ld_flow && !s1_prf 636 val s1_mmio = s1_mmio_cbo 637 638 s1_vaddr_hi := s1_in.vaddr(VAddrBits - 1, 6) 639 s1_vaddr_lo := s1_in.vaddr(5, 0) 640 s1_vaddr := Cat(s1_vaddr_hi, s1_vaddr_lo) 641 s1_paddr_dup_lsu := io.tlb.resp.bits.paddr(0) 642 s1_paddr_dup_dcache := io.tlb.resp.bits.paddr(1) 643 644 when (s1_tlb_memidx.is_ld && io.tlb.resp.valid && !s1_tlb_miss && 645 s1_tlb_memidx.idx === s1_in.uop.lqIdx.value && s1_ld_flow) { 646 // printf("Load idx = %d\n", s1_tlb_memidx.idx) 647 s1_out.uop.debugInfo.tlbRespTime := GTimer() 648 } .elsewhen(s1_tlb_memidx.is_st && io.tlb.resp.valid && !s1_tlb_miss && 649 s1_tlb_memidx.idx === s1_out.uop.sqIdx.value && !s1_ld_flow) { 650 // printf("Store idx = %d\n", s1_tlb_memidx.idx) 651 s1_out.uop.debugInfo.tlbRespTime := GTimer() 652 } 653 654 io.tlb.req_kill := s1_kill 655 io.tlb.resp.ready := true.B 656 657 io.ldu_io.dcache.s1_paddr_dup_lsu <> s1_paddr_dup_lsu 658 io.ldu_io.dcache.s1_paddr_dup_dcache <> s1_paddr_dup_dcache 659 io.ldu_io.dcache.s1_kill := s1_kill || s1_tlb_miss || s1_exception 660 661 // store to load forwarding 662 io.ldu_io.sbuffer.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_fast_rep_kill || s1_prf || !s1_ld_flow) 663 io.ldu_io.sbuffer.vaddr := s1_vaddr 664 io.ldu_io.sbuffer.paddr := s1_paddr_dup_lsu 665 io.ldu_io.sbuffer.uop := s1_in.uop 666 io.ldu_io.sbuffer.sqIdx := s1_in.uop.sqIdx 667 io.ldu_io.sbuffer.mask := s1_in.mask 668 io.ldu_io.sbuffer.pc := s1_in.uop.pc // FIXME: remove it 669 670 io.ldu_io.vec_forward.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_fast_rep_kill || s1_prf || !s1_ld_flow) 671 io.ldu_io.vec_forward.vaddr := s1_vaddr 672 io.ldu_io.vec_forward.paddr := s1_paddr_dup_lsu 673 io.ldu_io.vec_forward.uop := s1_in.uop 674 io.ldu_io.vec_forward.sqIdx := s1_in.uop.sqIdx 675 io.ldu_io.vec_forward.mask := s1_in.mask 676 io.ldu_io.vec_forward.pc := s1_in.uop.pc // FIXME: remove it 677 678 io.ldu_io.lsq.forward.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_fast_rep_kill || s1_prf || !s1_ld_flow) 679 io.ldu_io.lsq.forward.vaddr := s1_vaddr 680 io.ldu_io.lsq.forward.paddr := s1_paddr_dup_lsu 681 io.ldu_io.lsq.forward.uop := s1_in.uop 682 io.ldu_io.lsq.forward.sqIdx := s1_in.uop.sqIdx 683 io.ldu_io.lsq.forward.sqIdxMask := 0.U 684 io.ldu_io.lsq.forward.mask := s1_in.mask 685 io.ldu_io.lsq.forward.pc := s1_in.uop.pc // FIXME: remove it 686 687 // st-ld violation query 688 val s1_nuke = VecInit((0 until StorePipelineWidth).map(w => { 689 io.ldu_io.stld_nuke_query(w).valid && // query valid 690 isAfter(s1_in.uop.robIdx, io.ldu_io.stld_nuke_query(w).bits.robIdx) && // older store 691 // TODO: Fix me when vector instruction 692 (s1_paddr_dup_lsu(PAddrBits-1, 3) === io.ldu_io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 693 (s1_in.mask & io.ldu_io.stld_nuke_query(w).bits.mask).orR // data mask contain 694 })).asUInt.orR && !s1_tlb_miss && s1_ld_flow 695 696 s1_out := s1_in 697 s1_out.vaddr := s1_vaddr 698 s1_out.paddr := s1_paddr_dup_lsu 699 s1_out.tlbMiss := s1_tlb_miss 700 s1_out.ptwBack := io.tlb.resp.bits.ptwBack 701 s1_out.rsIdx := s1_in.rsIdx 702 s1_out.rep_info.debug := s1_in.uop.debugInfo 703 s1_out.rep_info.nuke := s1_nuke && !s1_sw_prf 704 s1_out.lateKill := s1_late_kill 705 706 when (s1_ld_flow) { 707 when (!s1_late_kill) { 708 // current ori test will cause the case of ldest == 0, below will be modifeid in the future. 709 // af & pf exception were modified 710 s1_out.uop.exceptionVec(loadPageFault) := io.tlb.resp.bits.excp(0).pf.ld 711 s1_out.uop.exceptionVec(loadAccessFault) := io.tlb.resp.bits.excp(0).af.ld 712 } .otherwise { 713 s1_out.uop.exceptionVec(loadAddrMisaligned) := false.B 714 s1_out.uop.exceptionVec(loadAccessFault) := s1_late_kill 715 } 716 } .otherwise { 717 s1_out.uop.exceptionVec(storePageFault) := io.tlb.resp.bits.excp(0).pf.st 718 s1_out.uop.exceptionVec(storeAccessFault) := io.tlb.resp.bits.excp(0).af.st 719 } 720 721 // pointer chasing 722 val s1_try_ptr_chasing = RegNext(s0_do_try_ptr_chasing, false.B) 723 val s1_ptr_chasing_vaddr = RegEnable(s0_ptr_chasing_vaddr, s0_do_try_ptr_chasing) 724 val s1_fu_op_type_not_ld = WireInit(false.B) 725 val s1_not_fast_match = WireInit(false.B) 726 val s1_addr_mismatch = WireInit(false.B) 727 val s1_addr_misaligned = WireInit(false.B) 728 val s1_ptr_chasing_canceled = WireInit(false.B) 729 val s1_cancel_ptr_chasing = WireInit(false.B) 730 731 s1_kill := s1_late_kill || 732 s1_cancel_ptr_chasing || 733 s1_in.uop.robIdx.needFlush(io.redirect) || 734 RegEnable(s0_kill, false.B, io.lsin.valid || io.ldu_io.replay.valid || io.ldu_io.l2l_fwd_in.valid || io.ldu_io.fast_rep_in.valid || io.vec_stu_io.in.valid) 735 736 if (EnableLoadToLoadForward) { 737 // Sometimes, we need to cancel the load-load forwarding. 738 // These can be put at S0 if timing is bad at S1. 739 // Case 0: CACHE_SET(base + offset) != CACHE_SET(base) (lowest 6-bit addition has an overflow) 740 s1_addr_mismatch := s1_ptr_chasing_vaddr(6) || RegEnable(io.ldu_io.ld_fast_imm(11, 6).orR, s0_do_try_ptr_chasing) 741 // Case 1: the address is misaligned, kill s1 742 s1_addr_misaligned := LookupTree(s1_in.uop.fuOpType(1, 0), List( 743 "b00".U -> false.B, //b 744 "b01".U -> (s1_vaddr(0) =/= 0.U), //h 745 "b10".U -> (s1_vaddr(1, 0) =/= 0.U), //w 746 "b11".U -> (s1_vaddr(2, 0) =/= 0.U) //d 747 )) 748 // Case 2: this load-load uop is cancelled 749 s1_ptr_chasing_canceled := !io.lsin.valid || FuType.isStore(io.lsin.bits.uop.fuType) 750 751 when (s1_try_ptr_chasing) { 752 s1_cancel_ptr_chasing := s1_addr_mismatch || s1_addr_misaligned || s1_ptr_chasing_canceled 753 754 s1_in.uop := io.lsin.bits.uop 755 s1_in.rsIdx := io.lsin.bits.iqIdx 756 s1_in.isFirstIssue := io.lsin.bits.isFirstIssue 757 s1_vaddr_lo := s1_ptr_chasing_vaddr(5, 0) 758 s1_paddr_dup_lsu := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 759 s1_paddr_dup_dcache := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 760 761 // recored tlb time when get the data to ensure the correctness of the latency calculation (although it should not record in here, because it does not use tlb) 762 s1_in.uop.debugInfo.tlbFirstReqTime := GTimer() 763 s1_in.uop.debugInfo.tlbRespTime := GTimer() 764 } 765 when (!s1_cancel_ptr_chasing) { 766 s0_ptr_chasing_canceled := s1_try_ptr_chasing && !io.ldu_io.replay.fire && !io.ldu_io.fast_rep_in.fire 767 when (s1_try_ptr_chasing) { 768 io.lsin.ready := true.B 769 } 770 } 771 } 772 773 // pre-calcuate sqIdx mask in s0, then send it to lsq in s1 for forwarding 774 val s1_sqIdx_mask = RegNext(UIntToMask(s0_out.uop.sqIdx.value, StoreQueueSize)) 775 // to enable load-load, sqIdxMask must be calculated based on lsin.uop 776 // If the timing here is not OK, load-load forwarding has to be disabled. 777 // Or we calculate sqIdxMask at RS?? 778 io.ldu_io.lsq.forward.sqIdxMask := s1_sqIdx_mask 779 if (EnableLoadToLoadForward) { 780 when (s1_try_ptr_chasing) { 781 io.ldu_io.lsq.forward.sqIdxMask := UIntToMask(io.lsin.bits.uop.sqIdx.value, StoreQueueSize) 782 } 783 } 784 785 io.ldu_io.forward_mshr.valid := s1_valid && s1_out.forward_tlDchannel && s1_ld_flow 786 io.ldu_io.forward_mshr.mshrid := s1_out.mshrid 787 io.ldu_io.forward_mshr.paddr := s1_out.paddr 788 789 io.stu_io.dcache.s1_kill := s1_tlb_miss || s1_exception || s1_mmio || s1_in.uop.robIdx.needFlush(io.redirect) 790 io.stu_io.dcache.s1_paddr := s1_paddr_dup_dcache 791 792 793 // load debug 794 XSDebug(s1_valid && s1_ld_flow, 795 p"S1: pc ${Hexadecimal(s1_out.uop.pc)}, lId ${Hexadecimal(s1_out.uop.lqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 796 p"paddr ${Hexadecimal(s1_out.paddr)}, mmio ${s1_out.mmio}\n") 797 798 // store debug 799 XSDebug(s1_valid && !s1_ld_flow, 800 p"S1: pc ${Hexadecimal(s1_out.uop.pc)}, lId ${Hexadecimal(s1_out.uop.sqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 801 p"paddr ${Hexadecimal(s1_out.paddr)}, mmio ${s1_out.mmio}\n") 802 803 // store out 804 io.stu_io.lsq.valid := s1_valid && !s1_ld_flow && !s1_prf && !s1_isvec 805 io.stu_io.lsq.bits := s1_out 806 io.stu_io.lsq.bits.miss := s1_tlb_miss 807 808 io.vec_stu_io.lsq.valid := s1_valid && !s1_ld_flow && !s1_prf && s1_isvec 809 io.vec_stu_io.lsq.bits := s1_out 810 io.vec_stu_io.lsq.bits.miss := s1_tlb_miss 811 io.vec_stu_io.lsq.bits.isLastElem := s1_isLastElem 812 813 io.stu_io.st_mask_out.valid := s1_valid && !s1_ld_flow && !s1_prf 814 io.stu_io.st_mask_out.bits.mask := s1_out.mask 815 io.stu_io.st_mask_out.bits.sqIdx := s1_out.uop.sqIdx 816 817 io.stu_io.issue.valid := s1_valid && !s1_tlb_miss && !s1_ld_flow && !s1_prf && !s1_isvec 818 io.stu_io.issue.bits := RegEnable(io.lsin.bits, io.lsin.fire) 819 820 // st-ld violation dectect request 821 io.stu_io.stld_nuke_query.valid := s1_valid && !s1_tlb_miss && !s1_ld_flow && !s1_prf 822 io.stu_io.stld_nuke_query.bits.robIdx := s1_in.uop.robIdx 823 io.stu_io.stld_nuke_query.bits.paddr := s1_paddr_dup_lsu 824 io.stu_io.stld_nuke_query.bits.mask := s1_in.mask 825 826 // Pipeline 827 // -------------------------------------------------------------------------------- 828 // stage 2 829 // -------------------------------------------------------------------------------- 830 // s2: DCache resp 831 val s2_valid = RegInit(false.B) 832 val s2_in = Wire(new LqWriteBundle) 833 val s2_out = Wire(new LqWriteBundle) 834 val s2_kill = Wire(Bool()) 835 val s2_can_go = s3_ready 836 val s2_fire = s2_valid && !s2_kill && s2_can_go 837 val s2_isvec = RegEnable(s1_isvec, false.B, s1_fire) 838 val s2_exp = RegEnable(s1_out.exp, true.B, s1_fire) 839 val s2_paddr = RegEnable(s1_paddr_dup_lsu, s1_fire) 840 841 s2_kill := s2_in.uop.robIdx.needFlush(io.redirect) 842 s2_ready := !s2_valid || s2_kill || s3_ready 843 when (s1_fire) { s2_valid := true.B } 844 .elsewhen (s2_fire) { s2_valid := false.B } 845 .elsewhen (s2_kill) { s2_valid := false.B } 846 s2_in := RegEnable(s1_out, s1_fire) 847 848 val s2_pmp = WireInit(io.pmp) 849 850 val s2_prf = s2_in.isPrefetch 851 val s2_hw_prf = s2_in.isHWPrefetch 852 val s2_ld_flow = RegEnable(s1_ld_flow, s1_fire) 853 854 // exception that may cause load addr to be invalid / illegal 855 // if such exception happen, that inst and its exception info 856 // will be force writebacked to rob 857 val s2_exception_vec = WireInit(s2_in.uop.exceptionVec) 858 when (s2_ld_flow) { 859 when (!s2_in.lateKill) { 860 s2_exception_vec(loadAccessFault) := (s2_in.uop.exceptionVec(loadAccessFault) || s2_pmp.ld) && s2_exp 861 // soft prefetch will not trigger any exception (but ecc error interrupt may be triggered) 862 when (s2_prf || s2_in.tlbMiss) { 863 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 864 } 865 } 866 } .otherwise { 867 s2_exception_vec(storeAccessFault) := s2_in.uop.exceptionVec(storeAccessFault) || s2_pmp.st 868 when (s2_prf || s2_in.tlbMiss) { 869 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 870 } 871 } 872 val s2_ld_exception = ExceptionNO.selectByFu(s2_exception_vec, LduCfg).asUInt.orR && s2_ld_flow 873 val s2_st_exception = ExceptionNO.selectByFu(s2_exception_vec, StaCfg).asUInt.orR && !s2_ld_flow 874 val s2_exception = s2_ld_exception || s2_st_exception 875 876 val (s2_fwd_frm_d_chan, s2_fwd_data_frm_d_chan) = io.ldu_io.tl_d_channel.forward(s1_valid && s1_out.forward_tlDchannel, s1_out.mshrid, s1_out.paddr) 877 val (s2_fwd_data_valid, s2_fwd_frm_mshr, s2_fwd_data_frm_mshr) = io.ldu_io.forward_mshr.forward() 878 val s2_fwd_frm_d_chan_or_mshr = s2_fwd_data_valid && (s2_fwd_frm_d_chan || s2_fwd_frm_mshr) 879 880 // writeback access fault caused by ecc error / bus error 881 // * ecc data error is slow to generate, so we will not use it until load stage 3 882 // * in load stage 3, an extra signal io.load_error will be used to 883 val s2_actually_mmio = s2_pmp.mmio 884 val s2_ld_mmio = !s2_prf && 885 s2_actually_mmio && 886 !s2_exception && 887 !s2_in.tlbMiss && 888 s2_ld_flow 889 val s2_st_mmio = !s2_prf && 890 (RegNext(s1_mmio) || s2_pmp.mmio) && 891 !s2_exception && 892 !s2_in.tlbMiss && 893 !s2_ld_flow 894 val s2_st_atomic = !s2_prf && 895 (RegNext(s1_mmio) || s2_pmp.atomic) && 896 !s2_exception && 897 !s2_in.tlbMiss && 898 !s2_ld_flow 899 val s2_full_fwd = Wire(Bool()) 900 val s2_mem_amb = s2_in.uop.storeSetHit && 901 io.ldu_io.lsq.forward.addrInvalid 902 903 val s2_tlb_miss = s2_in.tlbMiss 904 val s2_fwd_fail = io.ldu_io.lsq.forward.dataInvalid || io.ldu_io.vec_forward.dataInvalid 905 val s2_dcache_miss = io.ldu_io.dcache.resp.bits.miss && 906 !s2_fwd_frm_d_chan_or_mshr && 907 !s2_full_fwd 908 909 val s2_mq_nack = io.ldu_io.dcache.s2_mq_nack && 910 !s2_fwd_frm_d_chan_or_mshr && 911 !s2_full_fwd 912 913 val s2_bank_conflict = io.ldu_io.dcache.s2_bank_conflict && 914 !s2_fwd_frm_d_chan_or_mshr && 915 !s2_full_fwd 916 917 val s2_wpu_pred_fail = io.ldu_io.dcache.s2_wpu_pred_fail && 918 !s2_fwd_frm_d_chan_or_mshr && 919 !s2_full_fwd 920 921 val s2_rar_nack = io.ldu_io.lsq.ldld_nuke_query.req.valid && 922 !io.ldu_io.lsq.ldld_nuke_query.req.ready 923 924 val s2_raw_nack = io.ldu_io.lsq.stld_nuke_query.req.valid && 925 !io.ldu_io.lsq.stld_nuke_query.req.ready 926 927 // st-ld violation query 928 // NeedFastRecovery Valid when 929 // 1. Fast recovery query request Valid. 930 // 2. Load instruction is younger than requestors(store instructions). 931 // 3. Physical address match. 932 // 4. Data contains. 933 val s2_nuke = VecInit((0 until StorePipelineWidth).map(w => { 934 io.ldu_io.stld_nuke_query(w).valid && // query valid 935 isAfter(s2_in.uop.robIdx, io.ldu_io.stld_nuke_query(w).bits.robIdx) && // older store 936 // TODO: Fix me when vector instruction 937 (s2_in.paddr(PAddrBits-1, 3) === io.ldu_io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 938 (s2_in.mask & io.ldu_io.stld_nuke_query(w).bits.mask).orR // data mask contain 939 })).asUInt.orR && s2_ld_flow || s2_in.rep_info.nuke 940 941 val s2_cache_handled = io.ldu_io.dcache.resp.bits.handled 942 val s2_cache_tag_error = RegNext(io.csrCtrl.cache_error_enable) && 943 io.ldu_io.dcache.resp.bits.tag_error 944 945 val s2_troublem = !s2_exception && 946 !s2_ld_mmio && 947 !s2_prf && 948 !s2_in.lateKill && 949 s2_ld_flow 950 951 io.ldu_io.dcache.resp.ready := true.B 952 io.stu_io.dcache.resp.ready := true.B 953 val s2_dcache_should_resp = !(s2_in.tlbMiss || s2_exception || s2_ld_mmio || s2_prf || s2_in.lateKill) && s2_ld_flow 954 assert(!(s2_valid && (s2_dcache_should_resp && !io.ldu_io.dcache.resp.valid)), "DCache response got lost") 955 956 // fast replay require 957 val s2_dcache_fast_rep = (s2_mq_nack || !s2_dcache_miss && (s2_bank_conflict || s2_wpu_pred_fail)) 958 val s2_nuke_fast_rep = !s2_mq_nack && 959 !s2_dcache_miss && 960 !s2_bank_conflict && 961 !s2_wpu_pred_fail && 962 !s2_rar_nack && 963 !s2_raw_nack && 964 s2_nuke 965 966 val s2_fast_rep = !s2_mem_amb && 967 !s2_tlb_miss && 968 !s2_fwd_fail && 969 (s2_dcache_fast_rep || s2_nuke_fast_rep) && 970 s2_troublem 971 972 // need allocate new entry 973 val s2_can_query = !s2_mem_amb && 974 !s2_tlb_miss && 975 !s2_fwd_fail && 976 !s2_dcache_fast_rep && 977 s2_troublem 978 979 val s2_data_fwded = s2_dcache_miss && (s2_full_fwd || s2_cache_tag_error) 980 981 // ld-ld violation require 982 io.ldu_io.lsq.ldld_nuke_query.req.valid := s2_valid && s2_can_query 983 io.ldu_io.lsq.ldld_nuke_query.req.bits.uop := s2_in.uop 984 io.ldu_io.lsq.ldld_nuke_query.req.bits.mask := s2_in.mask 985 io.ldu_io.lsq.ldld_nuke_query.req.bits.paddr := s2_in.paddr 986 io.ldu_io.lsq.ldld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid, true.B, !s2_dcache_miss) 987 988 // st-ld violation require 989 io.ldu_io.lsq.stld_nuke_query.req.valid := s2_valid && s2_can_query 990 io.ldu_io.lsq.stld_nuke_query.req.bits.uop := s2_in.uop 991 io.ldu_io.lsq.stld_nuke_query.req.bits.mask := s2_in.mask 992 io.ldu_io.lsq.stld_nuke_query.req.bits.paddr := s2_in.paddr 993 io.ldu_io.lsq.stld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid, true.B, !s2_dcache_miss) 994 995 // merge forward result 996 // lsq has higher priority than sbuffer 997 val s2_fwd_mask = Wire(Vec((VLEN/8), Bool())) 998 val s2_fwd_data = Wire(Vec((VLEN/8), UInt(8.W))) 999 s2_full_fwd := ((~s2_fwd_mask.asUInt).asUInt & s2_in.mask) === 0.U && !io.ldu_io.lsq.forward.dataInvalid && !io.ldu_io.vec_forward.dataInvalid 1000 // generate XLEN/8 Muxs 1001 for (i <- 0 until VLEN / 8) { 1002 s2_fwd_mask(i) := io.ldu_io.lsq.forward.forwardMask(i) || io.ldu_io.sbuffer.forwardMask(i) || io.ldu_io.vec_forward.forwardMask(i) 1003 s2_fwd_data(i) := Mux( 1004 io.ldu_io.lsq.forward.forwardMask(i), 1005 io.ldu_io.lsq.forward.forwardData(i), 1006 Mux( 1007 io.ldu_io.vec_forward.forwardMask(i), 1008 io.ldu_io.vec_forward.forwardData(i), 1009 io.ldu_io.sbuffer.forwardData(i) 1010 ) 1011 ) 1012 } 1013 1014 XSDebug(s2_fire && s2_ld_flow, "[FWD LOAD RESP] pc %x fwd %x(%b) + %x(%b)\n", 1015 s2_in.uop.pc, 1016 io.ldu_io.lsq.forward.forwardData.asUInt, io.ldu_io.lsq.forward.forwardMask.asUInt, 1017 s2_in.forwardData.asUInt, s2_in.forwardMask.asUInt 1018 ) 1019 1020 // 1021 s2_out := s2_in 1022 s2_out.data := 0.U // data will be generated in load s3 1023 s2_out.uop.fpWen := s2_in.uop.fpWen && !s2_exception && s2_ld_flow 1024 s2_out.mmio := s2_ld_mmio || s2_st_mmio 1025 s2_out.atomic := s2_st_atomic 1026 s2_out.uop.flushPipe := false.B 1027 s2_out.uop.exceptionVec := s2_exception_vec 1028 s2_out.forwardMask := s2_fwd_mask 1029 s2_out.forwardData := s2_fwd_data 1030 s2_out.handledByMSHR := s2_cache_handled 1031 s2_out.miss := s2_dcache_miss && s2_troublem 1032 s2_out.feedbacked := io.feedback_fast.valid && !io.feedback_fast.bits.hit 1033 1034 // Generate replay signal caused by: 1035 // * st-ld violation check 1036 // * tlb miss 1037 // * dcache replay 1038 // * forward data invalid 1039 // * dcache miss 1040 s2_out.rep_info.mem_amb := s2_mem_amb && s2_troublem 1041 s2_out.rep_info.tlb_miss := s2_tlb_miss && s2_troublem 1042 s2_out.rep_info.fwd_fail := s2_fwd_fail && s2_troublem 1043 s2_out.rep_info.dcache_rep := s2_mq_nack && s2_troublem 1044 s2_out.rep_info.dcache_miss := s2_dcache_miss && s2_troublem 1045 s2_out.rep_info.bank_conflict := s2_bank_conflict && s2_troublem 1046 s2_out.rep_info.wpu_fail := s2_wpu_pred_fail && s2_troublem 1047 s2_out.rep_info.rar_nack := s2_rar_nack && s2_troublem 1048 s2_out.rep_info.raw_nack := s2_raw_nack && s2_troublem 1049 s2_out.rep_info.nuke := s2_nuke && s2_troublem 1050 s2_out.rep_info.full_fwd := s2_data_fwded 1051 s2_out.rep_info.data_inv_sq_idx := Mux(io.ldu_io.vec_forward.dataInvalid, s2_out.uop.sqIdx, io.ldu_io.lsq.forward.dataInvalidSqIdx) 1052 s2_out.rep_info.addr_inv_sq_idx := Mux(io.ldu_io.vec_forward.addrInvalid, s2_out.uop.sqIdx, io.ldu_io.lsq.forward.addrInvalidSqIdx) 1053 s2_out.rep_info.rep_carry := io.ldu_io.dcache.resp.bits.replayCarry 1054 s2_out.rep_info.mshr_id := io.ldu_io.dcache.resp.bits.mshr_id 1055 s2_out.rep_info.last_beat := s2_in.paddr(log2Up(refillBytes)) 1056 s2_out.rep_info.debug := s2_in.uop.debugInfo 1057 s2_out.rep_info.tlb_id := io.ldu_io.tlb_hint.id 1058 s2_out.rep_info.tlb_full := io.ldu_io.tlb_hint.full 1059 1060 // if forward fail, replay this inst from fetch 1061 val debug_fwd_fail_rep = s2_fwd_fail && !s2_troublem && !s2_in.tlbMiss 1062 // if ld-ld violation is detected, replay from this inst from fetch 1063 val debug_ldld_nuke_rep = false.B // s2_ldld_violation && !s2_ld_mmio && !s2_is_prefetch && !s2_in.tlbMiss 1064 // io.out.bits.uop.replayInst := false.B 1065 1066 // to be removed 1067 val s2_ld_need_fb = !s2_in.isLoadReplay && // already feedbacked 1068 io.ldu_io.lq_rep_full && // LoadQueueReplay is full 1069 s2_out.rep_info.need_rep && // need replay 1070 !s2_exception && // no exception is triggered 1071 !s2_hw_prf && // not hardware prefetch 1072 !s2_isvec 1073 val s2_st_need_fb = !s2_ld_flow && !s2_hw_prf && !s2_isvec 1074 io.feedback_fast.valid := s2_valid && (s2_ld_need_fb || s2_st_need_fb) 1075 io.feedback_fast.bits.hit := Mux(s2_ld_flow, false.B, !s2_tlb_miss) 1076 io.feedback_fast.bits.flushState := s2_in.ptwBack 1077 io.feedback_fast.bits.robIdx := s2_in.uop.robIdx 1078 io.feedback_fast.bits.sourceType := Mux(s2_ld_flow, RSFeedbackType.lrqFull, RSFeedbackType.tlbMiss) 1079 io.feedback_fast.bits.dataInvalidSqIdx := DontCare 1080 1081 val s2_vec_feedback = Wire(Valid(new VSFQFeedback)) 1082 s2_vec_feedback.valid := s2_valid && !s2_ld_flow && !s2_hw_prf && s2_isvec 1083 s2_vec_feedback.bits.flowPtr := s2_out.sflowPtr 1084 s2_vec_feedback.bits.hit := !s2_tlb_miss 1085 s2_vec_feedback.bits.sourceType := RSFeedbackType.tlbMiss 1086 s2_vec_feedback.bits.paddr := s2_paddr 1087 s2_vec_feedback.bits.mmio := s2_st_mmio 1088 s2_vec_feedback.bits.atomic := s2_st_mmio 1089 1090 io.stu_io.lsq_replenish := s2_out 1091 io.stu_io.lsq_replenish.miss := io.ldu_io.dcache.resp.fire && io.ldu_io.dcache.resp.bits.miss 1092 1093 io.ldu_io.ldCancel.ld1Cancel.valid := s2_valid && s2_ld_flow && s2_out.isFirstIssue && (// issued from IQ 1094 s2_out.rep_info.need_rep || s2_ld_mmio // exe fail or is mmio 1095 ) 1096 io.ldu_io.ldCancel.ld1Cancel.bits := s2_out.deqPortIdx 1097 1098 // fast wakeup 1099 io.ldu_io.fast_uop.valid := RegNext( 1100 !io.ldu_io.dcache.s1_disable_fast_wakeup && 1101 s1_valid && 1102 !s1_kill && 1103 !io.tlb.resp.bits.miss && 1104 !io.ldu_io.lsq.forward.dataInvalidFast 1105 ) && (s2_valid && !s2_out.rep_info.need_rep && !s2_ld_mmio && s2_ld_flow) && !s2_isvec 1106 io.ldu_io.fast_uop.bits := RegNext(s1_out.uop) 1107 1108 // 1109 io.ldu_io.s2_ptr_chasing := RegEnable(s1_try_ptr_chasing && !s1_cancel_ptr_chasing, false.B, s1_fire) 1110 1111 // prefetch train 1112 io.prefetch_train.valid := s2_valid && !s2_actually_mmio && !s2_in.tlbMiss 1113 io.prefetch_train.bits.fromLsPipelineBundle(s2_in) 1114 io.prefetch_train.bits.miss := Mux(s2_ld_flow, io.ldu_io.dcache.resp.bits.miss, io.stu_io.dcache.resp.bits.miss) // TODO: use trace with bank conflict? 1115 io.prefetch_train.bits.meta_prefetch := Mux(s2_ld_flow, io.ldu_io.dcache.resp.bits.meta_prefetch, false.B) 1116 io.prefetch_train.bits.meta_access := Mux(s2_ld_flow, io.ldu_io.dcache.resp.bits.meta_access, false.B) 1117 1118 io.prefetch_train_l1.valid := s2_valid && !s2_actually_mmio && s2_ld_flow 1119 io.prefetch_train_l1.bits.fromLsPipelineBundle(s2_in) 1120 io.prefetch_train_l1.bits.miss := io.ldu_io.dcache.resp.bits.miss 1121 io.prefetch_train_l1.bits.meta_prefetch := io.ldu_io.dcache.resp.bits.meta_prefetch 1122 io.prefetch_train_l1.bits.meta_access := io.ldu_io.dcache.resp.bits.meta_access 1123 if (env.FPGAPlatform){ 1124 io.ldu_io.dcache.s0_pc := DontCare 1125 io.ldu_io.dcache.s1_pc := DontCare 1126 io.ldu_io.dcache.s2_pc := DontCare 1127 }else{ 1128 io.ldu_io.dcache.s0_pc := s0_out.uop.pc 1129 io.ldu_io.dcache.s1_pc := s1_out.uop.pc 1130 io.ldu_io.dcache.s2_pc := s2_out.uop.pc 1131 } 1132 io.ldu_io.dcache.s2_kill := s2_pmp.ld || s2_actually_mmio || s2_kill 1133 io.stu_io.dcache.s2_kill := s2_pmp.st || s2_actually_mmio || s2_kill 1134 io.stu_io.dcache.s2_pc := s2_out.uop.pc 1135 1136 val s1_ld_left_fire = s1_valid && !s1_kill && s2_ready && s1_ld_flow 1137 val s2_ld_valid_dup = RegInit(0.U(6.W)) 1138 s2_ld_valid_dup := 0x0.U(6.W) 1139 when (s1_ld_left_fire && !s1_out.isHWPrefetch && s1_ld_flow) { s2_ld_valid_dup := 0x3f.U(6.W) } 1140 when (s1_kill || s1_out.isHWPrefetch || !s1_ld_flow) { s2_ld_valid_dup := 0x0.U(6.W) } 1141 assert(RegNext((s2_valid === s2_ld_valid_dup(0)) || RegNext(s1_out.isHWPrefetch) || RegNext(!s1_ld_flow))) 1142 1143 // Pipeline 1144 // -------------------------------------------------------------------------------- 1145 // stage 3 1146 // -------------------------------------------------------------------------------- 1147 // writeback and update load queue 1148 val s3_valid = RegNext(s2_valid && !s2_out.isHWPrefetch && !s2_out.uop.robIdx.needFlush(io.redirect)) 1149 val s3_in = RegEnable(s2_out, s2_fire) 1150 val s3_out = Wire(Valid(new MemExuOutput)) 1151 val s3_dcache_rep = RegEnable(s2_dcache_fast_rep && s2_troublem, false.B, s2_fire) 1152 val s3_ld_valid_dup = RegEnable(s2_ld_valid_dup, s2_fire) 1153 val s3_fast_rep = Wire(Bool()) 1154 val s3_ld_flow = RegNext(s2_ld_flow) 1155 val s3_troublem = RegNext(s2_troublem) 1156 val s3_kill = s3_in.uop.robIdx.needFlush(io.redirect) 1157 val s3_isvec = RegNext(s2_isvec) 1158 s3_ready := !s3_valid || s3_kill || sx_can_go 1159 1160 // forwrad last beat 1161 val (s3_fwd_frm_d_chan, s3_fwd_data_frm_d_chan) = io.ldu_io.tl_d_channel.forward(s2_valid && s2_out.forward_tlDchannel, s2_out.mshrid, s2_out.paddr) 1162 val s3_fwd_data_valid = RegEnable(s2_fwd_data_valid, false.B, s2_valid) 1163 val s3_fwd_frm_d_chan_valid = (s3_fwd_frm_d_chan && s3_fwd_data_valid) && s3_ld_flow 1164 1165 1166 // s3 load fast replay 1167 io.ldu_io.fast_rep_out.valid := s3_valid && 1168 s3_fast_rep && 1169 !s3_in.uop.robIdx.needFlush(io.redirect) && 1170 s3_ld_flow && 1171 !s3_isvec 1172 io.ldu_io.fast_rep_out.bits := s3_in 1173 1174 io.ldu_io.lsq.ldin.valid := s3_valid && 1175 (!s3_fast_rep || !io.ldu_io.fast_rep_out.ready) && 1176 !s3_in.feedbacked && 1177 !s3_in.lateKill && 1178 s3_ld_flow 1179 io.ldu_io.lsq.ldin.bits := s3_in 1180 io.ldu_io.lsq.ldin.bits.miss := s3_in.miss && !s3_fwd_frm_d_chan_valid 1181 1182 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1183 io.ldu_io.lsq.ldin.bits.data_wen_dup := s3_ld_valid_dup.asBools 1184 io.ldu_io.lsq.ldin.bits.replacementUpdated := io.ldu_io.dcache.resp.bits.replacementUpdated 1185 io.ldu_io.lsq.ldin.bits.missDbUpdated := RegNext(s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated) 1186 1187 val s3_dly_ld_err = 1188 if (EnableAccurateLoadError) { 1189 (s3_in.lateKill || io.ldu_io.dcache.resp.bits.error_delayed) && RegNext(io.csrCtrl.cache_error_enable) 1190 } else { 1191 WireInit(false.B) 1192 } 1193 io.ldu_io.s3_dly_ld_err := false.B // s3_dly_ld_err && s3_valid 1194 io.ldu_io.fast_rep_out.bits.delayedLoadError := s3_dly_ld_err 1195 io.ldu_io.lsq.ldin.bits.dcacheRequireReplay := s3_dcache_rep 1196 1197 val s3_vp_match_fail = RegNext(io.ldu_io.lsq.forward.matchInvalid || io.ldu_io.sbuffer.matchInvalid) && s3_troublem 1198 val s3_ldld_rep_inst = 1199 io.ldu_io.lsq.ldld_nuke_query.resp.valid && 1200 io.ldu_io.lsq.ldld_nuke_query.resp.bits.rep_frm_fetch && 1201 RegNext(io.csrCtrl.ldld_vio_check_enable) 1202 1203 val s3_rep_info = WireInit(s3_in.rep_info) 1204 s3_rep_info.dcache_miss := s3_in.rep_info.dcache_miss && !s3_fwd_frm_d_chan_valid && s3_troublem 1205 val s3_rep_frm_fetch = s3_vp_match_fail 1206 val s3_flushPipe = s3_ldld_rep_inst 1207 val s3_sel_rep_cause = PriorityEncoderOH(s3_rep_info.cause.asUInt) 1208 val s3_force_rep = s3_sel_rep_cause(LoadReplayCauses.C_TM) && 1209 !s3_in.uop.exceptionVec(loadAddrMisaligned) && 1210 s3_troublem 1211 1212 val s3_ld_exception = ExceptionNO.selectByFu(s3_in.uop.exceptionVec, LduCfg).asUInt.orR && s3_ld_flow 1213 val s3_st_exception = ExceptionNO.selectByFu(s3_in.uop.exceptionVec, StaCfg).asUInt.orR && !s3_ld_flow 1214 val s3_exception = s3_ld_exception || s3_st_exception 1215 when ((s3_ld_exception || s3_dly_ld_err || s3_rep_frm_fetch) && !s3_force_rep) { 1216 io.ldu_io.lsq.ldin.bits.rep_info.cause := 0.U.asTypeOf(s3_rep_info.cause.cloneType) 1217 } .otherwise { 1218 io.ldu_io.lsq.ldin.bits.rep_info.cause := VecInit(s3_sel_rep_cause.asBools) 1219 } 1220 1221 // Int flow, if hit, will be writebacked at s3 1222 s3_out.valid := s3_valid && 1223 (!s3_ld_flow && !s3_in.feedbacked || !io.ldu_io.lsq.ldin.bits.rep_info.need_rep) && !s3_in.mmio 1224 s3_out.bits.uop := s3_in.uop 1225 s3_out.bits.uop.exceptionVec(loadAccessFault) := (s3_dly_ld_err || s3_in.uop.exceptionVec(loadAccessFault)) && s3_ld_flow 1226 s3_out.bits.uop.replayInst := s3_rep_frm_fetch 1227 s3_out.bits.data := s3_in.data 1228 s3_out.bits.debug.isMMIO := s3_in.mmio 1229 s3_out.bits.debug.isPerfCnt := false.B 1230 s3_out.bits.debug.paddr := s3_in.paddr 1231 s3_out.bits.debug.vaddr := s3_in.vaddr 1232 1233 when (s3_force_rep) { 1234 s3_out.bits.uop.exceptionVec := 0.U.asTypeOf(s3_in.uop.exceptionVec.cloneType) 1235 } 1236 1237 io.ldu_io.rollback.valid := s3_valid && (s3_rep_frm_fetch || s3_flushPipe) && !s3_exception && s3_ld_flow 1238 io.ldu_io.rollback.bits := DontCare 1239 io.ldu_io.rollback.bits.isRVC := s3_out.bits.uop.preDecodeInfo.isRVC 1240 io.ldu_io.rollback.bits.robIdx := s3_out.bits.uop.robIdx 1241 io.ldu_io.rollback.bits.ftqIdx := s3_out.bits.uop.ftqPtr 1242 io.ldu_io.rollback.bits.ftqOffset := s3_out.bits.uop.ftqOffset 1243 io.ldu_io.rollback.bits.level := Mux(s3_rep_frm_fetch, RedirectLevel.flush, RedirectLevel.flushAfter) 1244 io.ldu_io.rollback.bits.cfiUpdate.target := s3_out.bits.uop.pc 1245 io.ldu_io.rollback.bits.debug_runahead_checkpoint_id := s3_out.bits.uop.debugInfo.runahead_checkpoint_id 1246 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1247 io.ldu_io.lsq.ldin.bits.uop := s3_out.bits.uop 1248 1249 val s3_revoke = s3_exception || io.ldu_io.lsq.ldin.bits.rep_info.need_rep 1250 io.ldu_io.lsq.ldld_nuke_query.revoke := s3_revoke 1251 io.ldu_io.lsq.stld_nuke_query.revoke := s3_revoke 1252 1253 // feedback slow 1254 s3_fast_rep := RegNext(s2_fast_rep) && 1255 !s3_in.feedbacked && 1256 !s3_in.lateKill && 1257 !s3_rep_frm_fetch && 1258 !s3_exception 1259 1260 val s3_fb_no_waiting = !s3_in.isLoadReplay && !(s3_fast_rep && io.ldu_io.fast_rep_out.ready) && !s3_in.feedbacked 1261 1262 // 1263 io.feedback_slow.valid := s3_valid && !s3_in.uop.robIdx.needFlush(io.redirect) && s3_fb_no_waiting && s3_ld_flow 1264 io.feedback_slow.bits.hit := !io.ldu_io.lsq.ldin.bits.rep_info.need_rep || io.ldu_io.lsq.ldin.ready 1265 io.feedback_slow.bits.flushState := s3_in.ptwBack 1266 io.feedback_slow.bits.robIdx := s3_in.uop.robIdx 1267 io.feedback_slow.bits.sourceType := RSFeedbackType.lrqFull 1268 io.feedback_slow.bits.dataInvalidSqIdx := DontCare 1269 1270 io.vec_stu_io.feedbackSlow.valid := RegNext(s2_vec_feedback.valid && !s2_out.uop.robIdx.needFlush(io.redirect)) 1271 io.vec_stu_io.feedbackSlow.bits := RegNext(s2_vec_feedback.bits) 1272 1273 io.ldu_io.ldCancel.ld2Cancel.valid := s3_valid && s3_ld_flow && s3_in.isFirstIssue && ( // issued from IQ and is load 1274 io.ldu_io.lsq.ldin.bits.rep_info.need_rep || s3_in.mmio // exe fail or is mmio 1275 ) 1276 io.ldu_io.ldCancel.ld2Cancel.bits := s3_in.deqPortIdx 1277 1278 // data from dcache hit 1279 val s3_ld_raw_data_frm_cache = Wire(new LoadDataFromDcacheBundle) 1280 s3_ld_raw_data_frm_cache.respDcacheData := io.ldu_io.dcache.resp.bits.data_delayed 1281 s3_ld_raw_data_frm_cache.forwardMask := RegEnable(s2_fwd_mask, s2_valid) 1282 s3_ld_raw_data_frm_cache.forwardData := RegEnable(s2_fwd_data, s2_valid) 1283 s3_ld_raw_data_frm_cache.uop := RegEnable(s2_out.uop, s2_valid) 1284 s3_ld_raw_data_frm_cache.addrOffset := RegEnable(s2_out.paddr(3, 0), s2_valid) 1285 s3_ld_raw_data_frm_cache.forward_D := RegEnable(s2_fwd_frm_d_chan, false.B, s2_valid) || s3_fwd_frm_d_chan_valid 1286 s3_ld_raw_data_frm_cache.forwardData_D := Mux(s3_fwd_frm_d_chan_valid, s3_fwd_data_frm_d_chan, RegEnable(s2_fwd_data_frm_d_chan, s2_valid)) 1287 s3_ld_raw_data_frm_cache.forward_mshr := RegEnable(s2_fwd_frm_mshr, false.B, s2_valid) 1288 s3_ld_raw_data_frm_cache.forwardData_mshr := RegEnable(s2_fwd_data_frm_mshr, s2_valid) 1289 s3_ld_raw_data_frm_cache.forward_result_valid := RegEnable(s2_fwd_data_valid, false.B, s2_valid) 1290 1291 val s3_merged_data_frm_cache = s3_ld_raw_data_frm_cache.mergedData() 1292 val s3_picked_data_frm_cache = LookupTree(s3_ld_raw_data_frm_cache.addrOffset, List( 1293 "b0000".U -> s3_merged_data_frm_cache(63, 0), 1294 "b0001".U -> s3_merged_data_frm_cache(63, 8), 1295 "b0010".U -> s3_merged_data_frm_cache(63, 16), 1296 "b0011".U -> s3_merged_data_frm_cache(63, 24), 1297 "b0100".U -> s3_merged_data_frm_cache(63, 32), 1298 "b0101".U -> s3_merged_data_frm_cache(63, 40), 1299 "b0110".U -> s3_merged_data_frm_cache(63, 48), 1300 "b0111".U -> s3_merged_data_frm_cache(63, 56), 1301 "b1000".U -> s3_merged_data_frm_cache(127, 64), 1302 "b1001".U -> s3_merged_data_frm_cache(127, 72), 1303 "b1010".U -> s3_merged_data_frm_cache(127, 80), 1304 "b1011".U -> s3_merged_data_frm_cache(127, 88), 1305 "b1100".U -> s3_merged_data_frm_cache(127, 96), 1306 "b1101".U -> s3_merged_data_frm_cache(127, 104), 1307 "b1110".U -> s3_merged_data_frm_cache(127, 112), 1308 "b1111".U -> s3_merged_data_frm_cache(127, 120) 1309 )) 1310 val s3_ld_data_frm_cache = rdataHelper(s3_ld_raw_data_frm_cache.uop, s3_picked_data_frm_cache) 1311 1312 // FIXME: add 1 cycle delay ? 1313 io.ldout.bits := s3_out.bits 1314 io.ldout.bits.data := s3_ld_data_frm_cache 1315 io.ldout.valid := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) && s3_ld_flow && !s3_isvec 1316 1317 // for uncache 1318 io.ldu_io.lsq.uncache.ready := true.B 1319 1320 // fast load to load forward 1321 if (EnableLoadToLoadForward) { 1322 io.ldu_io.l2l_fwd_out.valid := s3_out.valid && !s3_in.lateKill && s3_ld_flow 1323 io.ldu_io.l2l_fwd_out.data := s3_ld_data_frm_cache 1324 io.ldu_io.l2l_fwd_out.dly_ld_err := s3_dly_ld_err // ecc delayed error 1325 } else { 1326 io.ldu_io.l2l_fwd_out.valid := false.B 1327 io.ldu_io.l2l_fwd_out.data := DontCare 1328 io.ldu_io.l2l_fwd_out.dly_ld_err := DontCare 1329 } 1330 1331 // hybrid unit writeback to rob 1332 // delay params 1333 val SelectGroupSize = RollbackGroupSize 1334 val lgSelectGroupSize = log2Ceil(SelectGroupSize) 1335 val TotalSelectCycles = scala.math.ceil(log2Ceil(LoadQueueRAWSize).toFloat / lgSelectGroupSize).toInt + 1 1336 val TotalDelayCycles = TotalSelectCycles - 2 1337 1338 // writeback 1339 val sx_valid = Wire(Vec(TotalDelayCycles + 1, Bool())) 1340 val sx_ready = Wire(Vec(TotalDelayCycles + 1, Bool())) 1341 val sx_in = Wire(Vec(TotalDelayCycles + 1, new MemExuOutput)) 1342 1343 sx_can_go := sx_ready.head 1344 for (i <- 0 until TotalDelayCycles + 1) { 1345 if (i == 0) { 1346 sx_valid(i) := s3_valid && 1347 !s3_ld_flow && 1348 !s3_in.feedbacked && 1349 !s3_in.mmio 1350 sx_in(i) := s3_out.bits 1351 sx_ready(i) := !s3_valid(i) || sx_in(i).uop.robIdx.needFlush(io.redirect) || (if (TotalDelayCycles == 0) io.stout.ready else sx_ready(i+1)) 1352 } else { 1353 val cur_kill = sx_in(i).uop.robIdx.needFlush(io.redirect) 1354 val cur_can_go = (if (i == TotalDelayCycles) io.stout.ready else sx_ready(i+1)) 1355 val cur_fire = sx_valid(i) && !cur_kill && cur_can_go 1356 val prev_fire = sx_valid(i-1) && !sx_in(i-1).uop.robIdx.needFlush(io.redirect) && sx_ready(i) 1357 1358 sx_ready(i) := !sx_valid(i) || cur_kill || (if (i == TotalDelayCycles) io.stout.ready else sx_ready(i+1)) 1359 val sx_valid_can_go = prev_fire || cur_fire || cur_kill 1360 sx_valid(i) := RegEnable(Mux(prev_fire, true.B, false.B), sx_valid_can_go) 1361 sx_in(i) := RegEnable(sx_in(i-1), prev_fire) 1362 } 1363 } 1364 1365 val sx_last_valid = sx_valid.takeRight(1).head 1366 val sx_last_ready = sx_ready.takeRight(1).head 1367 val sx_last_in = sx_in.takeRight(1).head 1368 1369 sx_last_ready := !sx_last_valid || sx_last_in.uop.robIdx.needFlush(io.redirect) || io.stout.ready 1370 io.stout.valid := sx_last_valid && !sx_last_in.uop.robIdx.needFlush(io.redirect) && FuType.isStore(sx_last_in.uop.fuType) 1371 io.stout.bits := sx_last_in 1372 1373 // trigger 1374 val ld_trigger = FuType.isLoad(io.ldout.bits.uop.fuType) 1375 val last_valid_data = RegEnable(io.ldout.bits.data, io.stout.fire) 1376 val hit_ld_addr_trig_hit_vec = Wire(Vec(TriggerNum, Bool())) 1377 val lq_ld_addr_trig_hit_vec = RegNext(io.ldu_io.lsq.trigger.lqLoadAddrTriggerHitVec) 1378 (0 until TriggerNum).map{i => { 1379 val tdata2 = RegNext(RegNext(io.ldu_io.trigger(i).tdata2)) 1380 val matchType = RegNext(RegNext(io.ldu_io.trigger(i).matchType)) 1381 val tEnable = RegNext(RegNext(io.ldu_io.trigger(i).tEnable)) 1382 1383 hit_ld_addr_trig_hit_vec(i) := TriggerCmp(RegNext(s3_in.vaddr), tdata2, matchType, tEnable) 1384 io.ldu_io.trigger(i).addrHit := Mux(io.ldout.valid && ld_trigger, hit_ld_addr_trig_hit_vec(i), lq_ld_addr_trig_hit_vec(i)) 1385 io.ldu_io.trigger(i).lastDataHit := TriggerCmp(last_valid_data, tdata2, matchType, tEnable) 1386 }} 1387 io.ldu_io.lsq.trigger.hitLoadAddrTriggerHitVec := hit_ld_addr_trig_hit_vec 1388 1389 // FIXME: please move this part to LoadQueueReplay 1390 io.ldu_io.debug_ls := DontCare 1391 io.stu_io.debug_ls := DontCare 1392 io.stu_io.debug_ls.s1.isTlbFirstMiss := io.tlb.resp.valid && io.tlb.resp.bits.miss && io.tlb.resp.bits.debug.isFirstIssue && !s1_in.isHWPrefetch && !s1_ld_flow 1393 io.stu_io.debug_ls.s1_robIdx := s1_in.uop.robIdx.value 1394 1395 // Topdown 1396 io.ldu_io.lsTopdownInfo.s1.robIdx := s1_in.uop.robIdx.value 1397 io.ldu_io.lsTopdownInfo.s1.vaddr_valid := s1_valid && s1_in.hasROBEntry 1398 io.ldu_io.lsTopdownInfo.s1.vaddr_bits := s1_vaddr 1399 io.ldu_io.lsTopdownInfo.s2.robIdx := s2_in.uop.robIdx.value 1400 io.ldu_io.lsTopdownInfo.s2.paddr_valid := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss 1401 io.ldu_io.lsTopdownInfo.s2.paddr_bits := s2_in.paddr 1402 io.ldu_io.lsTopdownInfo.s2.first_real_miss := io.ldu_io.dcache.resp.bits.real_miss 1403 io.ldu_io.lsTopdownInfo.s2.cache_miss_en := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated 1404 1405 // perf cnt 1406 XSPerfAccumulate("s0_in_valid", io.lsin.valid) 1407 XSPerfAccumulate("s0_in_block", io.lsin.valid && !io.lsin.fire) 1408 XSPerfAccumulate("s0_in_fire_first_issue", s0_valid && s0_isFirstIssue) 1409 XSPerfAccumulate("s0_lsq_fire_first_issue", io.ldu_io.replay.fire) 1410 XSPerfAccumulate("s0_ldu_fire_first_issue", io.lsin.fire && s0_isFirstIssue) 1411 XSPerfAccumulate("s0_fast_replay_issue", io.ldu_io.fast_rep_in.fire) 1412 XSPerfAccumulate("s0_stall_out", s0_valid && !s0_can_go) 1413 XSPerfAccumulate("s0_stall_ld_dcache", s0_valid && !io.ldu_io.dcache.req.ready) 1414 XSPerfAccumulate("s0_stall_st_dcache", s0_valid && !io.stu_io.dcache.req.ready) 1415 XSPerfAccumulate("s0_addr_spec_success", s0_fire && s0_vaddr(VAddrBits-1, 12) === io.lsin.bits.src(0)(VAddrBits-1, 12)) 1416 XSPerfAccumulate("s0_addr_spec_failed", s0_fire && s0_vaddr(VAddrBits-1, 12) =/= io.lsin.bits.src(0)(VAddrBits-1, 12)) 1417 XSPerfAccumulate("s0_addr_spec_success_once", s0_fire && s0_vaddr(VAddrBits-1, 12) === io.lsin.bits.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 1418 XSPerfAccumulate("s0_addr_spec_failed_once", s0_fire && s0_vaddr(VAddrBits-1, 12) =/= io.lsin.bits.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 1419 XSPerfAccumulate("s0_forward_tl_d_channel", s0_out.forward_tlDchannel) 1420 XSPerfAccumulate("s0_hardware_prefetch_fire", s0_fire && s0_hw_prf_select) 1421 XSPerfAccumulate("s0_software_prefetch_fire", s0_fire && s0_prf && s0_int_iss_select) 1422 XSPerfAccumulate("s0_hardware_prefetch_blocked", io.ldu_io.prefetch_req.valid && !s0_hw_prf_select) 1423 XSPerfAccumulate("s0_hardware_prefetch_total", io.ldu_io.prefetch_req.valid) 1424 1425 XSPerfAccumulate("s1_in_valid", s1_valid) 1426 XSPerfAccumulate("s1_in_fire", s1_fire) 1427 XSPerfAccumulate("s1_in_fire_first_issue", s1_fire && s1_in.isFirstIssue) 1428 XSPerfAccumulate("s1_tlb_miss", s1_fire && s1_tlb_miss) 1429 XSPerfAccumulate("s1_tlb_miss_first_issue", s1_fire && s1_tlb_miss && s1_in.isFirstIssue) 1430 XSPerfAccumulate("s1_stall_out", s1_valid && !s1_can_go) 1431 XSPerfAccumulate("s1_late_kill", s1_valid && s1_fast_rep_kill) 1432 1433 XSPerfAccumulate("s2_in_valid", s2_valid) 1434 XSPerfAccumulate("s2_in_fire", s2_fire) 1435 XSPerfAccumulate("s2_in_fire_first_issue", s2_fire && s2_in.isFirstIssue) 1436 XSPerfAccumulate("s2_dcache_miss", s2_fire && io.ldu_io.dcache.resp.bits.miss) 1437 XSPerfAccumulate("s2_dcache_miss_first_issue", s2_fire && io.ldu_io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1438 XSPerfAccumulate("s2_dcache_real_miss_first_issue", s2_fire && io.ldu_io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1439 XSPerfAccumulate("s2_full_forward", s2_fire && s2_full_fwd) 1440 XSPerfAccumulate("s2_dcache_miss_full_forward", s2_fire && s2_dcache_miss) 1441 XSPerfAccumulate("s2_fwd_frm_d_can", s2_valid && s2_fwd_frm_d_chan) 1442 XSPerfAccumulate("s2_fwd_frm_d_chan_or_mshr", s2_valid && s2_fwd_frm_d_chan_or_mshr) 1443 XSPerfAccumulate("s2_stall_out", s2_fire && !s2_can_go) 1444 XSPerfAccumulate("s2_prefetch", s2_fire && s2_prf) 1445 XSPerfAccumulate("s2_prefetch_ignored", s2_fire && s2_prf && s2_mq_nack) // ignore prefetch for mshr full / miss req port conflict 1446 XSPerfAccumulate("s2_prefetch_miss", s2_fire && s2_prf && io.ldu_io.dcache.resp.bits.miss) // prefetch req miss in l1 1447 XSPerfAccumulate("s2_prefetch_hit", s2_fire && s2_prf && !io.ldu_io.dcache.resp.bits.miss) // prefetch req hit in l1 1448 XSPerfAccumulate("s2_prefetch_accept", s2_fire && s2_prf && io.ldu_io.dcache.resp.bits.miss && !s2_mq_nack) // prefetch a missed line in l1, and l1 accepted it 1449 XSPerfAccumulate("s2_forward_req", s2_fire && s2_in.forward_tlDchannel) 1450 XSPerfAccumulate("s2_successfully_forward_channel_D", s2_fire && s2_fwd_frm_d_chan && s2_fwd_data_valid) 1451 XSPerfAccumulate("s2_successfully_forward_mshr", s2_fire && s2_fwd_frm_mshr && s2_fwd_data_valid) 1452 1453 XSPerfAccumulate("s3_fwd_frm_d_chan", s3_valid && s3_fwd_frm_d_chan_valid) 1454 1455 XSPerfAccumulate("load_to_load_forward", s1_try_ptr_chasing && !s1_ptr_chasing_canceled) 1456 XSPerfAccumulate("load_to_load_forward_try", s1_try_ptr_chasing) 1457 XSPerfAccumulate("load_to_load_forward_fail", s1_cancel_ptr_chasing) 1458 XSPerfAccumulate("load_to_load_forward_fail_cancelled", s1_cancel_ptr_chasing && s1_ptr_chasing_canceled) 1459 XSPerfAccumulate("load_to_load_forward_fail_wakeup_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && s1_not_fast_match) 1460 XSPerfAccumulate("load_to_load_forward_fail_op_not_ld", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && s1_fu_op_type_not_ld) 1461 XSPerfAccumulate("load_to_load_forward_fail_addr_align", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && s1_addr_misaligned) 1462 XSPerfAccumulate("load_to_load_forward_fail_set_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && !s1_addr_misaligned && s1_addr_mismatch) 1463 1464 // bug lyq: some signals in perfEvents are no longer suitable for the current MemBlock design 1465 // hardware performance counter 1466 val perfEvents = Seq( 1467 ("load_s0_in_fire ", s0_fire ), 1468 ("load_to_load_forward ", s1_fire && s1_try_ptr_chasing && !s1_ptr_chasing_canceled ), 1469 ("stall_dcache ", s0_valid && s0_can_go && !io.ldu_io.dcache.req.ready ), 1470 ("load_s1_in_fire ", s0_fire ), 1471 ("load_s1_tlb_miss ", s1_fire && io.tlb.resp.bits.miss ), 1472 ("load_s2_in_fire ", s1_fire ), 1473 ("load_s2_dcache_miss ", s2_fire && io.ldu_io.dcache.resp.bits.miss ), 1474 ) 1475 generatePerfEvent() 1476}