1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.Bundles.{DynInst, MemExuInput, MemExuOutput} 27import xiangshan.backend.fu.PMPRespBundle 28import xiangshan.backend.fu.FuConfig._ 29import xiangshan.backend.ctrlblock.{DebugLsInfoBundle, LsTopdownInfo} 30import xiangshan.backend.rob.RobPtr 31import xiangshan.backend.ctrlblock.DebugLsInfoBundle 32import xiangshan.backend.fu.util.SdtrigExt 33 34import xiangshan.cache._ 35import xiangshan.cache.wpu.ReplayCarry 36import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp} 37import xiangshan.mem.mdp._ 38 39class LoadToLsqReplayIO(implicit p: Parameters) extends XSBundle with HasDCacheParameters { 40 // mshr refill index 41 val mshr_id = UInt(log2Up(cfg.nMissEntries).W) 42 // get full data from store queue and sbuffer 43 val full_fwd = Bool() 44 // wait for data from store inst's store queue index 45 val data_inv_sq_idx = new SqPtr 46 // wait for address from store queue index 47 val addr_inv_sq_idx = new SqPtr 48 // replay carry 49 val rep_carry = new ReplayCarry(nWays) 50 // data in last beat 51 val last_beat = Bool() 52 // replay cause 53 val cause = Vec(LoadReplayCauses.allCauses, Bool()) 54 // performance debug information 55 val debug = new PerfDebugInfo 56 57 // alias 58 def mem_amb = cause(LoadReplayCauses.C_MA) 59 def tlb_miss = cause(LoadReplayCauses.C_TM) 60 def fwd_fail = cause(LoadReplayCauses.C_FF) 61 def dcache_rep = cause(LoadReplayCauses.C_DR) 62 def dcache_miss = cause(LoadReplayCauses.C_DM) 63 def wpu_fail = cause(LoadReplayCauses.C_WF) 64 def bank_conflict = cause(LoadReplayCauses.C_BC) 65 def rar_nack = cause(LoadReplayCauses.C_RAR) 66 def raw_nack = cause(LoadReplayCauses.C_RAW) 67 def nuke = cause(LoadReplayCauses.C_NK) 68 def need_rep = cause.asUInt.orR 69} 70 71 72class LoadToLsqIO(implicit p: Parameters) extends XSBundle { 73 val ldin = DecoupledIO(new LqWriteBundle) 74 val uncache = Flipped(DecoupledIO(new MemExuOutput)) 75 val ld_raw_data = Input(new LoadDataFromLQBundle) 76 val forward = new PipeLoadForwardQueryIO 77 val stld_nuke_query = new LoadNukeQueryIO 78 val ldld_nuke_query = new LoadNukeQueryIO 79 val trigger = Flipped(new LqTriggerIO) 80} 81 82class LoadToLoadIO(implicit p: Parameters) extends XSBundle { 83 val valid = Bool() 84 val data = UInt(XLEN.W) // load to load fast path is limited to ld (64 bit) used as vaddr src1 only 85 val dly_ld_err = Bool() 86} 87 88class LoadUnitTriggerIO(implicit p: Parameters) extends XSBundle { 89 val tdata2 = Input(UInt(64.W)) 90 val matchType = Input(UInt(2.W)) 91 val tEnable = Input(Bool()) // timing is calculated before this 92 val addrHit = Output(Bool()) 93 val lastDataHit = Output(Bool()) 94} 95 96class LoadUnit(implicit p: Parameters) extends XSModule 97 with HasLoadHelper 98 with HasPerfEvents 99 with HasDCacheParameters 100 with HasCircularQueuePtrHelper 101 with HasVLSUParameters 102 with SdtrigExt 103{ 104 val io = IO(new Bundle() { 105 // control 106 val redirect = Flipped(ValidIO(new Redirect)) 107 val csrCtrl = Flipped(new CustomCSRCtrlIO) 108 109 // int issue path 110 val ldin = Flipped(Decoupled(new MemExuInput)) 111 val ldout = Decoupled(new MemExuOutput) 112 113 // vec issue path 114 val vecldin = Flipped(Decoupled(new VecLoadPipeBundle)) 115 val vecldout = Decoupled(new VecExuOutput) 116 val vecReplay = Decoupled(new LsPipelineBundle) 117 118 // data path 119 val tlb = new TlbRequestIO(2) 120 val pmp = Flipped(new PMPRespBundle()) // arrive same to tlb now 121 val dcache = new DCacheLoadIO 122 val sbuffer = new LoadForwardQueryIO 123 val vec_forward = new LoadForwardQueryIO // forward from vec store flow queue 124 val lsq = new LoadToLsqIO 125 val tl_d_channel = Input(new DcacheToLduForwardIO) 126 val forward_mshr = Flipped(new LduToMissqueueForwardIO) 127 val refill = Flipped(ValidIO(new Refill)) 128 val l2_hint = Input(Valid(new L2ToL1Hint)) 129 130 // fast wakeup 131 // TODO: implement vector fast wakeup 132 val fast_uop = ValidIO(new DynInst) // early wakeup signal generated in load_s1, send to RS in load_s2 133 134 // trigger 135 val trigger = Vec(TriggerNum, new LoadUnitTriggerIO) 136 137 138 // prefetch 139 val prefetch_train = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to sms 140 val prefetch_train_l1 = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to stream & stride 141 val prefetch_req = Flipped(ValidIO(new L1PrefetchReq)) // hardware prefetch to l1 cache req 142 val canAcceptLowConfPrefetch = Output(Bool()) 143 val canAcceptHighConfPrefetch = Output(Bool()) 144 145 // load to load fast path 146 val l2l_fwd_in = Input(new LoadToLoadIO) 147 val l2l_fwd_out = Output(new LoadToLoadIO) 148 149 val ld_fast_match = Input(Bool()) 150 val ld_fast_fuOpType = Input(UInt()) 151 val ld_fast_imm = Input(UInt(12.W)) 152 153 // rs feedback 154 val feedback_fast = ValidIO(new RSFeedback) // stage 2 155 val feedback_slow = ValidIO(new RSFeedback) // stage 3 156 val ldCancel = Output(new LoadCancelIO()) // use to cancel the uops waked by this load, and cancel load 157 158 // load ecc error 159 val s3_dly_ld_err = Output(Bool()) // Note that io.s3_dly_ld_err and io.lsq.s3_dly_ld_err is different 160 161 // schedule error query 162 val stld_nuke_query = Flipped(Vec(StorePipelineWidth, Valid(new StoreNukeQueryIO))) 163 164 // queue-based replay 165 val replay = Flipped(Decoupled(new LsPipelineBundle)) 166 val lq_rep_full = Input(Bool()) 167 168 // misc 169 val s2_ptr_chasing = Output(Bool()) // provide right pc for hw prefetch 170 171 // Load fast replay path 172 val fast_rep_in = Flipped(Decoupled(new LqWriteBundle)) 173 val fast_rep_out = Decoupled(new LqWriteBundle) 174 175 // perf 176 val debug_ls = Output(new DebugLsInfoBundle) 177 val lsTopdownInfo = Output(new LsTopdownInfo) 178 val correctMissTrain = Input(Bool()) 179 }) 180 181 val s1_ready, s2_ready, s3_ready = WireInit(false.B) 182 183 // Pipeline 184 // -------------------------------------------------------------------------------- 185 // stage 0 186 // -------------------------------------------------------------------------------- 187 // generate addr, use addr to query DCache and DTLB 188 val s0_valid = Wire(Bool()) 189 val s0_kill = Wire(Bool()) 190 val s0_vaddr = Wire(UInt(VAddrBits.W)) 191 val s0_mask = Wire(UInt((VLEN/8).W)) 192 val s0_uop = Wire(new DynInst) 193 val s0_has_rob_entry = Wire(Bool()) 194 val s0_rsIdx = Wire(UInt(log2Up(MemIQSizeMax).W)) 195 val s0_mshrid = Wire(UInt()) 196 val s0_try_l2l = Wire(Bool()) 197 val s0_rep_carry = Wire(new ReplayCarry(nWays)) 198 val s0_isFirstIssue = Wire(Bool()) 199 val s0_fast_rep = Wire(Bool()) 200 val s0_ld_rep = Wire(Bool()) 201 val s0_l2l_fwd = Wire(Bool()) 202 val s0_sched_idx = Wire(UInt()) 203 // Record the issue port idx of load issue queue. This signal is used by load cancel. 204 val s0_deqPortIdx = Wire(UInt(log2Ceil(LoadPipelineWidth).W)) 205 val s0_can_go = s1_ready 206 val s0_fire = s0_valid && s0_can_go 207 val s0_out = Wire(new LqWriteBundle) 208 209 // vector related ctrl signal 210 val s0_isvec = WireInit(false.B) 211 val s0_is128bit = WireInit(false.B) 212 val s0_uop_unit_stride_fof = WireInit(false.B) 213 // val s0_rob_idx_valid = WireInit(VecInit(Seq.fill(2)(false.B))) 214 // val s0_inner_idx = WireInit(VecInit(Seq.fill(2)(0.U(3.W)))) 215 // val s0_rob_idx = WireInit(VecInit(Seq.fill(2)(0.U.asTypeOf(new RobPtr)))) 216 val s0_reg_offset = WireInit(0.U(vOffsetBits.W)) 217 // val s0_offset = WireInit(VecInit(Seq.fill(2)(0.U(4.W)))) 218 val s0_exp = WireInit(true.B) 219 val s0_is_first_ele = WireInit(false.B) 220 val s0_flowPtr = WireInit(0.U.asTypeOf(new VlflowPtr)) 221 222 // load flow select/gen 223 // src0: super load replayed by LSQ (cache miss replay) (io.replay) 224 // src1: fast load replay (io.fast_rep_in) 225 // src2: load replayed by LSQ (io.replay) 226 // src3: hardware prefetch from prefetchor (high confidence) (io.prefetch) 227 // src4: int read / software prefetch first issue from RS (io.in) 228 // src5: vec read from RS (io.vecldin) 229 // src6: load try pointchaising when no issued or replayed load (io.fastpath) 230 // src7: hardware prefetch from prefetchor (high confidence) (io.prefetch) 231 // priority: high to low 232 val s0_rep_stall = io.ldin.valid && isAfter(io.replay.bits.uop.robIdx, io.ldin.bits.uop.robIdx) 233 val s0_super_ld_rep_valid = io.replay.valid && io.replay.bits.forward_tlDchannel 234 val s0_ld_fast_rep_valid = io.fast_rep_in.valid 235 val s0_ld_rep_valid = io.replay.valid && !io.replay.bits.forward_tlDchannel && !s0_rep_stall 236 val s0_high_conf_prf_valid = io.prefetch_req.valid && io.prefetch_req.bits.confidence > 0.U 237 val s0_int_iss_valid = io.ldin.valid // int flow first issue or software prefetch 238 val s0_vec_iss_valid = io.vecldin.valid 239 val s0_l2l_fwd_valid = io.l2l_fwd_in.valid && io.ld_fast_match 240 val s0_low_conf_prf_valid = io.prefetch_req.valid && io.prefetch_req.bits.confidence === 0.U 241 dontTouch(s0_super_ld_rep_valid) 242 dontTouch(s0_ld_fast_rep_valid) 243 dontTouch(s0_ld_rep_valid) 244 dontTouch(s0_high_conf_prf_valid) 245 dontTouch(s0_int_iss_valid) 246 dontTouch(s0_vec_iss_valid) 247 dontTouch(s0_l2l_fwd_valid) 248 dontTouch(s0_low_conf_prf_valid) 249 250 // load flow source ready 251 val s0_super_ld_rep_ready = WireInit(true.B) 252 val s0_ld_fast_rep_ready = !s0_super_ld_rep_valid 253 val s0_ld_rep_ready = !s0_super_ld_rep_valid && 254 !s0_ld_fast_rep_valid 255 val s0_high_conf_prf_ready = !s0_super_ld_rep_valid && 256 !s0_ld_fast_rep_valid && 257 !s0_ld_rep_valid 258 259 val s0_int_iss_ready = !s0_super_ld_rep_valid && 260 !s0_ld_fast_rep_valid && 261 !s0_ld_rep_valid && 262 !s0_high_conf_prf_valid 263 264 val s0_vec_iss_ready = !s0_super_ld_rep_valid && 265 !s0_ld_fast_rep_valid && 266 !s0_ld_rep_valid && 267 !s0_high_conf_prf_valid && 268 !s0_int_iss_valid 269 270 val s0_l2l_fwd_ready = !s0_super_ld_rep_valid && 271 !s0_ld_fast_rep_valid && 272 !s0_ld_rep_valid && 273 !s0_high_conf_prf_valid && 274 !s0_int_iss_valid && 275 !s0_vec_iss_valid 276 277 val s0_low_conf_prf_ready = !s0_super_ld_rep_valid && 278 !s0_ld_fast_rep_valid && 279 !s0_ld_rep_valid && 280 !s0_high_conf_prf_valid && 281 !s0_int_iss_valid && 282 !s0_vec_iss_valid && 283 !s0_l2l_fwd_valid 284 dontTouch(s0_super_ld_rep_ready) 285 dontTouch(s0_ld_fast_rep_ready) 286 dontTouch(s0_ld_rep_ready) 287 dontTouch(s0_high_conf_prf_ready) 288 dontTouch(s0_int_iss_ready) 289 dontTouch(s0_vec_iss_ready) 290 dontTouch(s0_l2l_fwd_ready) 291 dontTouch(s0_low_conf_prf_ready) 292 293 // load flow source select (OH) 294 val s0_super_ld_rep_select = s0_super_ld_rep_valid && s0_super_ld_rep_ready 295 val s0_ld_fast_rep_select = s0_ld_fast_rep_valid && s0_ld_fast_rep_ready 296 val s0_ld_rep_select = s0_ld_rep_valid && s0_ld_rep_ready 297 val s0_hw_prf_select = s0_high_conf_prf_ready && s0_high_conf_prf_valid || 298 s0_low_conf_prf_ready && s0_low_conf_prf_valid 299 val s0_int_iss_select = s0_int_iss_ready && s0_int_iss_valid 300 val s0_vec_iss_select = s0_vec_iss_ready && s0_vec_iss_valid 301 val s0_l2l_fwd_select = s0_l2l_fwd_ready && s0_l2l_fwd_valid 302 dontTouch(s0_super_ld_rep_select) 303 dontTouch(s0_ld_fast_rep_select) 304 dontTouch(s0_ld_rep_select) 305 dontTouch(s0_hw_prf_select) 306 dontTouch(s0_int_iss_select) 307 dontTouch(s0_vec_iss_select) 308 dontTouch(s0_l2l_fwd_select) 309 310 s0_valid := (s0_super_ld_rep_valid || 311 s0_ld_fast_rep_valid || 312 s0_ld_rep_valid || 313 s0_high_conf_prf_valid || 314 s0_int_iss_valid || 315 s0_vec_iss_valid || 316 s0_l2l_fwd_valid || 317 s0_low_conf_prf_valid) && io.dcache.req.ready && !s0_kill 318 319 // which is S0's out is ready and dcache is ready 320 val s0_try_ptr_chasing = s0_l2l_fwd_select 321 val s0_do_try_ptr_chasing = s0_try_ptr_chasing && s0_can_go && io.dcache.req.ready 322 val s0_ptr_chasing_vaddr = io.l2l_fwd_in.data(5, 0) +& io.ld_fast_imm(5, 0) 323 val s0_ptr_chasing_canceled = WireInit(false.B) 324 s0_kill := s0_ptr_chasing_canceled || (s0_out.uop.robIdx.needFlush(io.redirect) && !s0_try_ptr_chasing) 325 326 // prefetch related ctrl signal 327 val s0_prf = Wire(Bool()) 328 val s0_prf_rd = Wire(Bool()) 329 val s0_prf_wr = Wire(Bool()) 330 val s0_hw_prf = s0_hw_prf_select 331 332 io.canAcceptLowConfPrefetch := s0_low_conf_prf_ready 333 io.canAcceptHighConfPrefetch := s0_high_conf_prf_ready 334 335 // query DTLB 336 io.tlb.req.valid := s0_valid 337 io.tlb.req.bits.cmd := Mux(s0_prf, 338 Mux(s0_prf_wr, TlbCmd.write, TlbCmd.read), 339 TlbCmd.read 340 ) 341 io.tlb.req.bits.vaddr := Mux(s0_hw_prf_select, io.prefetch_req.bits.paddr, s0_vaddr) 342 io.tlb.req.bits.size := Mux(s0_isvec, io.vecldin.bits.alignedType, LSUOpType.size(s0_uop.fuOpType)) 343 io.tlb.req.bits.kill := s0_kill 344 io.tlb.req.bits.memidx.is_ld := true.B 345 io.tlb.req.bits.memidx.is_st := false.B 346 io.tlb.req.bits.memidx.idx := s0_uop.lqIdx.value 347 io.tlb.req.bits.debug.robIdx := s0_uop.robIdx 348 io.tlb.req.bits.no_translate := s0_hw_prf_select // hw b.reqetch addr does not need to be translated 349 io.tlb.req.bits.debug.pc := s0_uop.pc 350 io.tlb.req.bits.debug.isFirstIssue := s0_isFirstIssue 351 352 // query DCache 353 io.dcache.req.valid := s0_valid 354 io.dcache.req.bits.cmd := Mux(s0_prf_rd, 355 MemoryOpConstants.M_PFR, 356 Mux(s0_prf_wr, MemoryOpConstants.M_PFW, MemoryOpConstants.M_XRD) 357 ) 358 io.dcache.req.bits.vaddr := s0_vaddr 359 io.dcache.req.bits.mask := s0_mask 360 io.dcache.req.bits.data := DontCare 361 io.dcache.req.bits.isFirstIssue := s0_isFirstIssue 362 io.dcache.req.bits.instrtype := Mux(s0_prf, DCACHE_PREFETCH_SOURCE.U, LOAD_SOURCE.U) 363 io.dcache.req.bits.debug_robIdx := s0_uop.robIdx.value 364 io.dcache.req.bits.replayCarry := s0_rep_carry 365 // io.dcache.req.bits.is128bit := s0_is128bit 366 io.dcache.req.bits.id := DontCare // TODO: update cache meta 367 io.dcache.pf_source := Mux(s0_hw_prf_select, io.prefetch_req.bits.pf_source.value, L1_HW_PREFETCH_NULL) 368 369 // load flow priority mux 370 def fromNullSource() = { 371 s0_vaddr := 0.U 372 s0_mask := 0.U 373 s0_uop := 0.U.asTypeOf(new DynInst) 374 s0_try_l2l := false.B 375 s0_has_rob_entry := false.B 376 s0_rsIdx := 0.U 377 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 378 s0_mshrid := 0.U 379 s0_isFirstIssue := false.B 380 s0_fast_rep := false.B 381 s0_ld_rep := false.B 382 s0_l2l_fwd := false.B 383 s0_prf := false.B 384 s0_prf_rd := false.B 385 s0_prf_wr := false.B 386 s0_sched_idx := 0.U 387 s0_deqPortIdx := 0.U 388 } 389 390 def fromFastReplaySource(src: LqWriteBundle) = { 391 s0_vaddr := src.vaddr 392 s0_mask := src.mask 393 s0_uop := src.uop 394 s0_try_l2l := false.B 395 s0_has_rob_entry := src.hasROBEntry 396 s0_rep_carry := src.rep_info.rep_carry 397 s0_mshrid := src.rep_info.mshr_id 398 s0_rsIdx := src.rsIdx 399 s0_isFirstIssue := false.B 400 s0_fast_rep := true.B 401 s0_ld_rep := src.isLoadReplay 402 s0_l2l_fwd := false.B 403 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 404 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 405 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 406 s0_sched_idx := src.schedIndex 407 s0_deqPortIdx := src.deqPortIdx 408 } 409 410 def fromNormalReplaySource(src: LsPipelineBundle) = { 411 s0_vaddr := src.vaddr 412 s0_mask := genVWmask(src.vaddr, src.uop.fuOpType(1, 0)) 413 s0_uop := src.uop 414 s0_try_l2l := false.B 415 s0_has_rob_entry := true.B 416 s0_rsIdx := src.rsIdx 417 s0_rep_carry := src.replayCarry 418 s0_mshrid := src.mshrid 419 s0_isFirstIssue := false.B 420 s0_fast_rep := false.B 421 s0_ld_rep := true.B 422 s0_l2l_fwd := false.B 423 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 424 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 425 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 426 s0_sched_idx := src.schedIndex 427 s0_deqPortIdx := src.deqPortIdx 428 } 429 430 def fromPrefetchSource(src: L1PrefetchReq) = { 431 s0_vaddr := src.getVaddr() 432 s0_mask := 0.U 433 s0_uop := DontCare 434 s0_try_l2l := false.B 435 s0_has_rob_entry := false.B 436 s0_rsIdx := 0.U 437 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 438 s0_mshrid := 0.U 439 s0_isFirstIssue := false.B 440 s0_fast_rep := false.B 441 s0_ld_rep := false.B 442 s0_l2l_fwd := false.B 443 s0_prf := true.B 444 s0_prf_rd := !src.is_store 445 s0_prf_wr := src.is_store 446 s0_sched_idx := 0.U 447 s0_deqPortIdx := 0.U 448 } 449 450 def fromIntIssueSource(src: MemExuInput) = { 451 s0_vaddr := src.src(0) + SignExt(src.uop.imm(11, 0), VAddrBits) 452 s0_mask := genVWmask(s0_vaddr, src.uop.fuOpType(1,0)) 453 s0_uop := src.uop 454 s0_try_l2l := false.B 455 s0_has_rob_entry := true.B 456 s0_rsIdx := src.iqIdx 457 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 458 s0_mshrid := 0.U 459 s0_isFirstIssue := true.B 460 s0_fast_rep := false.B 461 s0_ld_rep := false.B 462 s0_l2l_fwd := false.B 463 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 464 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 465 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 466 s0_sched_idx := 0.U 467 s0_deqPortIdx := src.deqPortIdx 468 } 469 470 def fromVecIssueSource(src: VecLoadPipeBundle) = { 471 s0_vaddr := src.vaddr 472 s0_mask := src.mask 473 s0_uop := src.uop 474 s0_try_l2l := false.B 475 s0_has_rob_entry := true.B 476 // TODO: VLSU, implement vector feedback 477 s0_rsIdx := 0.U 478 // TODO: VLSU, implement replay carry 479 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 480 s0_mshrid := 0.U 481 // TODO: VLSU, implement first issue 482 s0_isFirstIssue := src.isFirstIssue 483 s0_fast_rep := false.B 484 s0_ld_rep := false.B 485 s0_l2l_fwd := false.B 486 s0_prf := false.B 487 s0_prf_rd := false.B 488 s0_prf_wr := false.B 489 s0_sched_idx := 0.U 490 // Vector load interface 491 s0_isvec := true.B 492 // vector loads only access a single element at a time, so 128-bit path is not used for now 493 s0_is128bit := false.B 494 s0_uop_unit_stride_fof := src.uop_unit_stride_fof 495 // s0_rob_idx_valid := src.rob_idx_valid 496 // s0_inner_idx := src.inner_idx 497 // s0_rob_idx := src.rob_idx 498 s0_reg_offset := src.reg_offset 499 // s0_offset := src.offset 500 s0_exp := src.exp 501 s0_is_first_ele := src.is_first_ele 502 s0_flowPtr := src.flowPtr 503 s0_deqPortIdx := 0.U 504 } 505 506 def fromLoadToLoadSource(src: LoadToLoadIO) = { 507 s0_vaddr := Cat(src.data(XLEN-1, 6), s0_ptr_chasing_vaddr(5,0)) 508 s0_mask := genVWmask(s0_vaddr, io.ld_fast_fuOpType(1, 0)) 509 // When there's no valid instruction from RS and LSQ, we try the load-to-load forwarding. 510 // Assume the pointer chasing is always ld. 511 s0_uop.fuOpType := io.ld_fast_fuOpType 512 s0_try_l2l := true.B 513 // we dont care s0_isFirstIssue and s0_rsIdx and s0_sqIdx and s0_deqPortIdx in S0 when trying pointchasing 514 // because these signals will be updated in S1 515 s0_has_rob_entry := false.B 516 s0_rsIdx := 0.U 517 s0_mshrid := 0.U 518 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 519 s0_isFirstIssue := true.B 520 s0_fast_rep := false.B 521 s0_ld_rep := false.B 522 s0_l2l_fwd := true.B 523 s0_prf := false.B 524 s0_prf_rd := false.B 525 s0_prf_wr := false.B 526 s0_sched_idx := 0.U 527 s0_deqPortIdx := 0.U 528 } 529 530 // set default 531 s0_uop := DontCare 532 when (s0_super_ld_rep_select) { fromNormalReplaySource(io.replay.bits) } 533 .elsewhen (s0_ld_fast_rep_select) { fromFastReplaySource(io.fast_rep_in.bits) } 534 .elsewhen (s0_ld_rep_select) { fromNormalReplaySource(io.replay.bits) } 535 .elsewhen (s0_hw_prf_select) { fromPrefetchSource(io.prefetch_req.bits) } 536 .elsewhen (s0_int_iss_select) { fromIntIssueSource(io.ldin.bits) } 537 .elsewhen (s0_vec_iss_select) { fromVecIssueSource(io.vecldin.bits) } 538 .otherwise { 539 if (EnableLoadToLoadForward) { 540 fromLoadToLoadSource(io.l2l_fwd_in) 541 } else { 542 fromNullSource() 543 } 544 } 545 546 // address align check 547 val s0_addr_aligned = LookupTree(Mux(s0_isvec, io.vecldin.bits.alignedType, s0_uop.fuOpType(1, 0)), List( 548 "b00".U -> true.B, //b 549 "b01".U -> (s0_vaddr(0) === 0.U), //h 550 "b10".U -> (s0_vaddr(1, 0) === 0.U), //w 551 "b11".U -> (s0_vaddr(2, 0) === 0.U) //d 552 )) 553 554 // accept load flow if dcache ready (tlb is always ready) 555 // TODO: prefetch need writeback to loadQueueFlag 556 s0_out := DontCare 557 s0_out.rsIdx := s0_rsIdx 558 s0_out.vaddr := s0_vaddr 559 s0_out.mask := s0_mask 560 s0_out.uop := s0_uop 561 s0_out.isFirstIssue := s0_isFirstIssue 562 s0_out.hasROBEntry := s0_has_rob_entry 563 s0_out.isPrefetch := s0_prf 564 s0_out.isHWPrefetch := s0_hw_prf 565 s0_out.isFastReplay := s0_fast_rep 566 s0_out.isLoadReplay := s0_ld_rep 567 s0_out.isFastPath := s0_l2l_fwd 568 s0_out.mshrid := s0_mshrid 569 s0_out.isvec := s0_isvec 570 s0_out.is128bit := s0_is128bit 571 s0_out.uop_unit_stride_fof := s0_uop_unit_stride_fof 572 // s0_out.rob_idx_valid := s0_rob_idx_valid 573 // s0_out.inner_idx := s0_inner_idx 574 // s0_out.rob_idx := s0_rob_idx 575 s0_out.reg_offset := s0_reg_offset 576 // s0_out.offset := s0_offset 577 s0_out.exp := s0_exp 578 s0_out.is_first_ele := s0_is_first_ele 579 s0_out.flowPtr := s0_flowPtr 580 s0_out.uop.exceptionVec(loadAddrMisaligned) := !s0_addr_aligned && s0_exp 581 s0_out.forward_tlDchannel := s0_super_ld_rep_select 582 when(io.tlb.req.valid && s0_isFirstIssue) { 583 s0_out.uop.debugInfo.tlbFirstReqTime := GTimer() 584 }.otherwise{ 585 s0_out.uop.debugInfo.tlbFirstReqTime := s0_uop.debugInfo.tlbFirstReqTime 586 } 587 s0_out.schedIndex := s0_sched_idx 588 s0_out.deqPortIdx := s0_deqPortIdx 589 590 // load fast replay 591 io.fast_rep_in.ready := (s0_can_go && io.dcache.req.ready && s0_ld_fast_rep_ready) 592 593 // load flow source ready 594 // cache missed load has highest priority 595 // always accept cache missed load flow from load replay queue 596 io.replay.ready := (s0_can_go && io.dcache.req.ready && (s0_ld_rep_ready && !s0_rep_stall || s0_super_ld_rep_select)) 597 598 // accept load flow from rs when: 599 // 1) there is no lsq-replayed load 600 // 2) there is no fast replayed load 601 // 3) there is no high confidence prefetch request 602 io.ldin.ready := s0_can_go && io.dcache.req.ready && s0_int_iss_ready 603 io.vecldin.ready := s0_can_go && io.dcache.req.ready && s0_vec_iss_ready 604 605 // for hw prefetch load flow feedback, to be added later 606 // io.prefetch_in.ready := s0_hw_prf_select 607 608 // dcache replacement extra info 609 // TODO: should prefetch load update replacement? 610 io.dcache.replacementUpdated := Mux(s0_ld_rep_select || s0_super_ld_rep_select, io.replay.bits.replacementUpdated, false.B) 611 612 XSDebug(io.dcache.req.fire, 613 p"[DCACHE LOAD REQ] pc ${Hexadecimal(s0_uop.pc)}, vaddr ${Hexadecimal(s0_vaddr)}\n" 614 ) 615 XSDebug(s0_valid, 616 p"S0: pc ${Hexadecimal(s0_out.uop.pc)}, lId ${Hexadecimal(s0_out.uop.lqIdx.asUInt)}, " + 617 p"vaddr ${Hexadecimal(s0_out.vaddr)}, mask ${Hexadecimal(s0_out.mask)}\n") 618 619 // Pipeline 620 // -------------------------------------------------------------------------------- 621 // stage 1 622 // -------------------------------------------------------------------------------- 623 // TLB resp (send paddr to dcache) 624 val s1_valid = RegInit(false.B) 625 val s1_in = Wire(new LqWriteBundle) 626 val s1_out = Wire(new LqWriteBundle) 627 val s1_kill = Wire(Bool()) 628 val s1_can_go = s2_ready 629 val s1_fire = s1_valid && !s1_kill && s1_can_go 630 val s1_exp = RegEnable(s0_out.exp, true.B, s0_fire) 631 val s1_isvec = RegEnable(s0_out.isvec, false.B, s0_fire) 632 val s1_vec_alignedType = RegEnable(io.vecldin.bits.alignedType, s0_fire) 633 634 s1_ready := !s1_valid || s1_kill || s2_ready 635 when (s0_fire) { s1_valid := true.B } 636 .elsewhen (s1_fire) { s1_valid := false.B } 637 .elsewhen (s1_kill) { s1_valid := false.B } 638 s1_in := RegEnable(s0_out, s0_fire) 639 640 val s1_fast_rep_dly_err = RegNext(io.fast_rep_in.bits.delayedLoadError) 641 val s1_fast_rep_kill = s1_fast_rep_dly_err && s1_in.isFastReplay 642 val s1_l2l_fwd_dly_err = RegNext(io.l2l_fwd_in.dly_ld_err) 643 val s1_l2l_fwd_kill = s1_l2l_fwd_dly_err && s1_in.isFastPath 644 val s1_late_kill = s1_fast_rep_kill || s1_l2l_fwd_kill 645 val s1_vaddr_hi = Wire(UInt()) 646 val s1_vaddr_lo = Wire(UInt()) 647 val s1_vaddr = Wire(UInt()) 648 val s1_paddr_dup_lsu = Wire(UInt()) 649 val s1_paddr_dup_dcache = Wire(UInt()) 650 val s1_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, LduCfg).asUInt.orR // af & pf exception were modified below. 651 val s1_tlb_miss = io.tlb.resp.bits.miss 652 val s1_prf = s1_in.isPrefetch 653 val s1_hw_prf = s1_in.isHWPrefetch 654 val s1_sw_prf = s1_prf && !s1_hw_prf 655 val s1_tlb_memidx = io.tlb.resp.bits.memidx 656 657 s1_vaddr_hi := s1_in.vaddr(VAddrBits - 1, 6) 658 s1_vaddr_lo := s1_in.vaddr(5, 0) 659 s1_vaddr := Cat(s1_vaddr_hi, s1_vaddr_lo) 660 s1_paddr_dup_lsu := io.tlb.resp.bits.paddr(0) 661 s1_paddr_dup_dcache := io.tlb.resp.bits.paddr(1) 662 663 when (s1_tlb_memidx.is_ld && io.tlb.resp.valid && !s1_tlb_miss && s1_tlb_memidx.idx === s1_in.uop.lqIdx.value) { 664 // printf("load idx = %d\n", s1_tlb_memidx.idx) 665 s1_out.uop.debugInfo.tlbRespTime := GTimer() 666 } 667 668 io.tlb.req_kill := s1_kill 669 io.tlb.resp.ready := true.B 670 671 io.dcache.s1_paddr_dup_lsu <> s1_paddr_dup_lsu 672 io.dcache.s1_paddr_dup_dcache <> s1_paddr_dup_dcache 673 io.dcache.s1_kill := s1_kill || s1_tlb_miss || s1_exception 674 675 // store to load forwarding 676 io.sbuffer.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_prf) 677 io.sbuffer.vaddr := s1_vaddr 678 io.sbuffer.paddr := s1_paddr_dup_lsu 679 io.sbuffer.uop := s1_in.uop 680 io.sbuffer.sqIdx := s1_in.uop.sqIdx 681 io.sbuffer.mask := s1_in.mask 682 io.sbuffer.pc := s1_in.uop.pc // FIXME: remove it 683 684 io.vec_forward.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_prf) 685 io.vec_forward.vaddr := s1_vaddr 686 io.vec_forward.paddr := s1_paddr_dup_lsu 687 io.vec_forward.uop := s1_in.uop 688 io.vec_forward.sqIdx := s1_in.uop.sqIdx 689 io.vec_forward.mask := s1_in.mask 690 io.vec_forward.pc := s1_in.uop.pc // FIXME: remove it 691 692 io.lsq.forward.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_prf) 693 io.lsq.forward.vaddr := s1_vaddr 694 io.lsq.forward.paddr := s1_paddr_dup_lsu 695 io.lsq.forward.uop := s1_in.uop 696 io.lsq.forward.sqIdx := s1_in.uop.sqIdx 697 io.lsq.forward.sqIdxMask := 0.U 698 io.lsq.forward.mask := s1_in.mask 699 io.lsq.forward.pc := s1_in.uop.pc // FIXME: remove it 700 701 // st-ld violation query 702 // val s1_nuke_paddr_match = VecInit((0 until StorePipelineWidth).map(w => {Mux(s1_isvec && s1_in.is128bit, 703 // s1_paddr_dup_lsu(PAddrBits-1, 4) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 4), 704 // s1_paddr_dup_lsu(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3))})) 705 val s1_nuke = VecInit((0 until StorePipelineWidth).map(w => { 706 io.stld_nuke_query(w).valid && // query valid 707 isAfter(s1_in.uop.robIdx, io.stld_nuke_query(w).bits.robIdx) && // older store 708 (s1_paddr_dup_lsu(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 709 (s1_in.mask & io.stld_nuke_query(w).bits.mask).orR // data mask contain 710 })).asUInt.orR && !s1_tlb_miss 711 712 s1_out := s1_in 713 s1_out.vaddr := s1_vaddr 714 s1_out.paddr := s1_paddr_dup_lsu 715 s1_out.tlbMiss := s1_tlb_miss 716 s1_out.ptwBack := io.tlb.resp.bits.ptwBack 717 s1_out.rsIdx := s1_in.rsIdx 718 s1_out.rep_info.debug := s1_in.uop.debugInfo 719 s1_out.rep_info.nuke := s1_nuke && !s1_sw_prf 720 s1_out.lateKill := s1_late_kill 721 722 when (!s1_late_kill) { 723 // current ori test will cause the case of ldest == 0, below will be modifeid in the future. 724 // af & pf exception were modified 725 s1_out.uop.exceptionVec(loadPageFault) := io.tlb.resp.bits.excp(0).pf.ld && s1_exp 726 s1_out.uop.exceptionVec(loadAccessFault) := io.tlb.resp.bits.excp(0).af.ld && s1_exp 727 } .otherwise { 728 s1_out.uop.exceptionVec(loadAddrMisaligned) := false.B && s1_exp 729 s1_out.uop.exceptionVec(loadAccessFault) := s1_late_kill && s1_exp 730 } 731 732 // pointer chasing 733 val s1_try_ptr_chasing = RegNext(s0_do_try_ptr_chasing, false.B) 734 val s1_ptr_chasing_vaddr = RegEnable(s0_ptr_chasing_vaddr, s0_do_try_ptr_chasing) 735 val s1_fu_op_type_not_ld = WireInit(false.B) 736 val s1_not_fast_match = WireInit(false.B) 737 val s1_addr_mismatch = WireInit(false.B) 738 val s1_addr_misaligned = WireInit(false.B) 739 val s1_ptr_chasing_canceled = WireInit(false.B) 740 val s1_cancel_ptr_chasing = WireInit(false.B) 741 742 s1_kill := s1_late_kill || 743 s1_cancel_ptr_chasing || 744 s1_in.uop.robIdx.needFlush(io.redirect) || 745 RegEnable(s0_kill, false.B, io.ldin.valid || io.replay.valid || io.l2l_fwd_in.valid || io.fast_rep_in.valid || io.vecldin.valid) 746 747 if (EnableLoadToLoadForward) { 748 // Sometimes, we need to cancel the load-load forwarding. 749 // These can be put at S0 if timing is bad at S1. 750 // Case 0: CACHE_SET(base + offset) != CACHE_SET(base) (lowest 6-bit addition has an overflow) 751 s1_addr_mismatch := s1_ptr_chasing_vaddr(6) || RegEnable(io.ld_fast_imm(11, 6).orR, s0_do_try_ptr_chasing) 752 // Case 1: the address is misaligned, kill s1 753 s1_addr_misaligned := LookupTree(s1_in.uop.fuOpType(1, 0), List( 754 "b00".U -> false.B, //b 755 "b01".U -> (s1_vaddr(0) =/= 0.U), //h 756 "b10".U -> (s1_vaddr(1, 0) =/= 0.U), //w 757 "b11".U -> (s1_vaddr(2, 0) =/= 0.U) //d 758 )) 759 // Case 2: this load-load uop is cancelled 760 s1_ptr_chasing_canceled := !io.ldin.valid 761 762 when (s1_try_ptr_chasing) { 763 s1_cancel_ptr_chasing := s1_addr_mismatch || s1_addr_misaligned || s1_ptr_chasing_canceled 764 765 s1_in.uop := io.ldin.bits.uop 766 s1_in.rsIdx := io.ldin.bits.iqIdx 767 s1_in.isFirstIssue := io.ldin.bits.isFirstIssue 768 s1_in.deqPortIdx := io.ldin.bits.deqPortIdx 769 s1_vaddr_lo := s1_ptr_chasing_vaddr(5, 0) 770 s1_paddr_dup_lsu := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 771 s1_paddr_dup_dcache := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 772 773 // recored tlb time when get the data to ensure the correctness of the latency calculation (although it should not record in here, because it does not use tlb) 774 s1_in.uop.debugInfo.tlbFirstReqTime := GTimer() 775 s1_in.uop.debugInfo.tlbRespTime := GTimer() 776 } 777 when (!s1_cancel_ptr_chasing) { 778 s0_ptr_chasing_canceled := s1_try_ptr_chasing && !io.replay.fire && !io.fast_rep_in.fire 779 when (s1_try_ptr_chasing) { 780 io.ldin.ready := true.B 781 } 782 } 783 } 784 785 // pre-calcuate sqIdx mask in s0, then send it to lsq in s1 for forwarding 786 val s1_sqIdx_mask = RegNext(UIntToMask(s0_out.uop.sqIdx.value, StoreQueueSize)) 787 // to enable load-load, sqIdxMask must be calculated based on ldin.uop 788 // If the timing here is not OK, load-load forwarding has to be disabled. 789 // Or we calculate sqIdxMask at RS?? 790 io.lsq.forward.sqIdxMask := s1_sqIdx_mask 791 if (EnableLoadToLoadForward) { 792 when (s1_try_ptr_chasing) { 793 io.lsq.forward.sqIdxMask := UIntToMask(io.ldin.bits.uop.sqIdx.value, StoreQueueSize) 794 } 795 } 796 797 io.forward_mshr.valid := s1_valid && s1_out.forward_tlDchannel 798 io.forward_mshr.mshrid := s1_out.mshrid 799 io.forward_mshr.paddr := s1_out.paddr 800 801 XSDebug(s1_valid, 802 p"S1: pc ${Hexadecimal(s1_out.uop.pc)}, lId ${Hexadecimal(s1_out.uop.lqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 803 p"paddr ${Hexadecimal(s1_out.paddr)}, mmio ${s1_out.mmio}\n") 804 805 // Pipeline 806 // -------------------------------------------------------------------------------- 807 // stage 2 808 // -------------------------------------------------------------------------------- 809 // s2: DCache resp 810 val s2_valid = RegInit(false.B) 811 val s2_in = Wire(new LqWriteBundle) 812 val s2_out = Wire(new LqWriteBundle) 813 val s2_kill = Wire(Bool()) 814 val s2_can_go = s3_ready 815 val s2_fire = s2_valid && !s2_kill && s2_can_go 816 val s2_exp = RegEnable(s1_out.exp, true.B, s1_fire) 817 val s2_isvec = RegEnable(s1_out.isvec, false.B, s1_fire) 818 val s2_vec_alignedType = RegEnable(s1_vec_alignedType, s1_fire) 819 820 s2_kill := s2_in.uop.robIdx.needFlush(io.redirect) 821 s2_ready := !s2_valid || s2_kill || s3_ready 822 when (s1_fire) { s2_valid := true.B } 823 .elsewhen (s2_fire) { s2_valid := false.B } 824 .elsewhen (s2_kill) { s2_valid := false.B } 825 s2_in := RegEnable(s1_out, s1_fire) 826 827 val s2_pmp = WireInit(io.pmp) 828 829 val s2_prf = s2_in.isPrefetch 830 val s2_hw_prf = s2_in.isHWPrefetch 831 832 // exception that may cause load addr to be invalid / illegal 833 // if such exception happen, that inst and its exception info 834 // will be force writebacked to rob 835 val s2_exception_vec = WireInit(s2_in.uop.exceptionVec) 836 when (!s2_in.lateKill) { 837 s2_exception_vec(loadAccessFault) := (s2_in.uop.exceptionVec(loadAccessFault) || s2_pmp.ld) && s2_exp 838 // soft prefetch will not trigger any exception (but ecc error interrupt may be triggered) 839 when (s2_prf || s2_in.tlbMiss) { 840 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 841 } 842 } 843 val s2_exception = ExceptionNO.selectByFu(s2_exception_vec, LduCfg).asUInt.orR && s2_exp 844 845 val (s2_fwd_frm_d_chan, s2_fwd_data_frm_d_chan) = io.tl_d_channel.forward(s1_valid && s1_out.forward_tlDchannel, s1_out.mshrid, s1_out.paddr) 846 val (s2_fwd_data_valid, s2_fwd_frm_mshr, s2_fwd_data_frm_mshr) = io.forward_mshr.forward() 847 val s2_fwd_frm_d_chan_or_mshr = s2_fwd_data_valid && (s2_fwd_frm_d_chan || s2_fwd_frm_mshr) 848 849 // writeback access fault caused by ecc error / bus error 850 // * ecc data error is slow to generate, so we will not use it until load stage 3 851 // * in load stage 3, an extra signal io.load_error will be used to 852 val s2_actually_mmio = s2_pmp.mmio 853 val s2_mmio = !s2_prf && 854 s2_actually_mmio && 855 !s2_exception && 856 !s2_in.tlbMiss 857 858 val s2_full_fwd = Wire(Bool()) 859 val s2_mem_amb = s2_in.uop.storeSetHit && 860 io.lsq.forward.addrInvalid 861 862 val s2_tlb_miss = s2_in.tlbMiss 863 val s2_fwd_fail = io.lsq.forward.dataInvalid || io.vec_forward.dataInvalid 864 val s2_dcache_miss = io.dcache.resp.bits.miss && 865 !s2_fwd_frm_d_chan_or_mshr && 866 !s2_full_fwd 867 868 val s2_mq_nack = io.dcache.s2_mq_nack && 869 !s2_fwd_frm_d_chan_or_mshr && 870 !s2_full_fwd 871 872 val s2_bank_conflict = io.dcache.s2_bank_conflict && 873 !s2_fwd_frm_d_chan_or_mshr && 874 !s2_full_fwd 875 876 val s2_wpu_pred_fail = io.dcache.s2_wpu_pred_fail && 877 !s2_fwd_frm_d_chan_or_mshr && 878 !s2_full_fwd 879 880 val s2_rar_nack = io.lsq.ldld_nuke_query.req.valid && 881 !io.lsq.ldld_nuke_query.req.ready 882 883 val s2_raw_nack = io.lsq.stld_nuke_query.req.valid && 884 !io.lsq.stld_nuke_query.req.ready 885 // st-ld violation query 886 // NeedFastRecovery Valid when 887 // 1. Fast recovery query request Valid. 888 // 2. Load instruction is younger than requestors(store instructions). 889 // 3. Physical address match. 890 // 4. Data contains. 891 val s2_nuke = VecInit((0 until StorePipelineWidth).map(w => { 892 io.stld_nuke_query(w).valid && // query valid 893 isAfter(s2_in.uop.robIdx, io.stld_nuke_query(w).bits.robIdx) && // older store 894 // TODO: Fix me when vector instruction 895 (s2_in.paddr(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 896 (s2_in.mask & io.stld_nuke_query(w).bits.mask).orR // data mask contain 897 })).asUInt.orR && !s2_tlb_miss || s2_in.rep_info.nuke 898 899 val s2_cache_handled = io.dcache.resp.bits.handled 900 val s2_cache_tag_error = RegNext(io.csrCtrl.cache_error_enable) && 901 io.dcache.resp.bits.tag_error 902 903 val s2_troublem = !s2_exception && 904 !s2_mmio && 905 !s2_prf && 906 !s2_in.lateKill 907 908 io.dcache.resp.ready := true.B 909 val s2_dcache_should_resp = !(s2_in.tlbMiss || s2_exception || s2_mmio || s2_prf || s2_in.lateKill) 910 assert(!(s2_valid && (s2_dcache_should_resp && !io.dcache.resp.valid)), "DCache response got lost") 911 912 // fast replay require 913 val s2_dcache_fast_rep = (s2_mq_nack || !s2_dcache_miss && (s2_bank_conflict || s2_wpu_pred_fail)) 914 val s2_nuke_fast_rep = !s2_mq_nack && 915 !s2_dcache_miss && 916 !s2_bank_conflict && 917 !s2_wpu_pred_fail && 918 !s2_rar_nack && 919 !s2_raw_nack && 920 s2_nuke 921 922 val s2_fast_rep = !s2_mem_amb && 923 !s2_tlb_miss && 924 !s2_fwd_fail && 925 (s2_dcache_fast_rep || s2_nuke_fast_rep) && 926 s2_troublem 927 928 // need allocate new entry 929 val s2_can_query = !s2_mem_amb && 930 !s2_tlb_miss && 931 !s2_fwd_fail && 932 !s2_dcache_fast_rep && 933 s2_troublem 934 935 val s2_data_fwded = s2_dcache_miss && (s2_full_fwd || s2_cache_tag_error) 936 937 // ld-ld violation require 938 io.lsq.ldld_nuke_query.req.valid := s2_valid && s2_can_query 939 io.lsq.ldld_nuke_query.req.bits.uop := s2_in.uop 940 io.lsq.ldld_nuke_query.req.bits.mask := s2_in.mask 941 io.lsq.ldld_nuke_query.req.bits.paddr := s2_in.paddr 942 io.lsq.ldld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid, true.B, !s2_dcache_miss) 943 944 // st-ld violation require 945 io.lsq.stld_nuke_query.req.valid := s2_valid && s2_can_query 946 io.lsq.stld_nuke_query.req.bits.uop := s2_in.uop 947 io.lsq.stld_nuke_query.req.bits.mask := s2_in.mask 948 io.lsq.stld_nuke_query.req.bits.paddr := s2_in.paddr 949 io.lsq.stld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid, true.B, !s2_dcache_miss) 950 951 // merge forward result 952 // lsq has higher priority than sbuffer 953 val s2_fwd_mask = Wire(Vec((VLEN/8), Bool())) 954 val s2_fwd_data = Wire(Vec((VLEN/8), UInt(8.W))) 955 s2_full_fwd := ((~s2_fwd_mask.asUInt).asUInt & s2_in.mask) === 0.U && !io.lsq.forward.dataInvalid && !io.vec_forward.dataInvalid 956 // generate XLEN/8 Muxs 957 for (i <- 0 until VLEN / 8) { 958 s2_fwd_mask(i) := io.lsq.forward.forwardMask(i) || io.sbuffer.forwardMask(i) || io.vec_forward.forwardMask(i) 959 s2_fwd_data(i) := Mux( 960 io.lsq.forward.forwardMask(i), 961 io.lsq.forward.forwardData(i), 962 Mux( 963 io.vec_forward.forwardMask(i), 964 io.vec_forward.forwardData(i), 965 io.sbuffer.forwardData(i) 966 ) 967 ) 968 } 969 970 XSDebug(s2_fire, "[FWD LOAD RESP] pc %x fwd %x(%b) + %x(%b)\n", 971 s2_in.uop.pc, 972 io.lsq.forward.forwardData.asUInt, io.lsq.forward.forwardMask.asUInt, 973 s2_in.forwardData.asUInt, s2_in.forwardMask.asUInt 974 ) 975 976 // 977 s2_out := s2_in 978 s2_out.data := 0.U // data will be generated in load s3 979 s2_out.uop.fpWen := s2_in.uop.fpWen && !s2_exception 980 s2_out.mmio := s2_mmio 981 s2_out.uop.flushPipe := false.B 982 s2_out.uop.exceptionVec := s2_exception_vec 983 s2_out.forwardMask := s2_fwd_mask 984 s2_out.forwardData := s2_fwd_data 985 s2_out.handledByMSHR := s2_cache_handled 986 s2_out.miss := s2_dcache_miss && s2_troublem 987 s2_out.feedbacked := io.feedback_fast.valid 988 989 // Generate replay signal caused by: 990 // * st-ld violation check 991 // * tlb miss 992 // * dcache replay 993 // * forward data invalid 994 // * dcache miss 995 s2_out.rep_info.mem_amb := s2_mem_amb && s2_troublem 996 s2_out.rep_info.tlb_miss := s2_tlb_miss && s2_troublem 997 s2_out.rep_info.fwd_fail := s2_fwd_fail && s2_troublem 998 s2_out.rep_info.dcache_rep := s2_mq_nack && s2_troublem 999 s2_out.rep_info.dcache_miss := s2_dcache_miss && s2_troublem 1000 s2_out.rep_info.bank_conflict := s2_bank_conflict && s2_troublem 1001 s2_out.rep_info.wpu_fail := s2_wpu_pred_fail && s2_troublem 1002 s2_out.rep_info.rar_nack := s2_rar_nack && s2_troublem 1003 s2_out.rep_info.raw_nack := s2_raw_nack && s2_troublem 1004 s2_out.rep_info.nuke := s2_nuke && s2_troublem 1005 s2_out.rep_info.full_fwd := s2_data_fwded 1006 s2_out.rep_info.data_inv_sq_idx := Mux(io.vec_forward.dataInvalid, s2_out.uop.sqIdx, io.lsq.forward.dataInvalidSqIdx) 1007 s2_out.rep_info.addr_inv_sq_idx := Mux(io.vec_forward.addrInvalid, s2_out.uop.sqIdx, io.lsq.forward.addrInvalidSqIdx) 1008 s2_out.rep_info.rep_carry := io.dcache.resp.bits.replayCarry 1009 s2_out.rep_info.mshr_id := io.dcache.resp.bits.mshr_id 1010 s2_out.rep_info.last_beat := s2_in.paddr(log2Up(refillBytes)) 1011 s2_out.rep_info.debug := s2_in.uop.debugInfo 1012 1013 // if forward fail, replay this inst from fetch 1014 val debug_fwd_fail_rep = s2_fwd_fail && !s2_troublem && !s2_in.tlbMiss 1015 // if ld-ld violation is detected, replay from this inst from fetch 1016 val debug_ldld_nuke_rep = false.B // s2_ldld_violation && !s2_mmio && !s2_is_prefetch && !s2_in.tlbMiss 1017 // io.out.bits.uop.replayInst := false.B 1018 1019 // to be removed 1020 io.feedback_fast.valid := s2_valid && // inst is valid 1021 !s2_in.isLoadReplay && // already feedbacked 1022 io.lq_rep_full && // LoadQueueReplay is full 1023 s2_out.rep_info.need_rep && // need replay 1024 !s2_exception && // no exception is triggered 1025 !s2_hw_prf && // not hardware prefetch 1026 !s2_isvec // not vector 1027 io.feedback_fast.bits.hit := false.B 1028 io.feedback_fast.bits.flushState := s2_in.ptwBack 1029 io.feedback_fast.bits.robIdx := s2_in.uop.robIdx 1030 io.feedback_fast.bits.sourceType := RSFeedbackType.lrqFull 1031 io.feedback_fast.bits.dataInvalidSqIdx := DontCare 1032 1033 io.ldCancel.ld1Cancel.valid := s2_valid && s2_out.isFirstIssue && ( // issued from IQ 1034 s2_out.rep_info.need_rep || s2_mmio // exe fail or is mmio 1035 ) 1036 io.ldCancel.ld1Cancel.bits := s2_out.deqPortIdx 1037 1038 // fast wakeup 1039 io.fast_uop.valid := RegNext( 1040 !io.dcache.s1_disable_fast_wakeup && 1041 s1_valid && 1042 !s1_kill && 1043 !io.tlb.resp.bits.miss && 1044 !io.lsq.forward.dataInvalidFast 1045 ) && (s2_valid && !s2_out.rep_info.need_rep && !s2_mmio) && !s2_isvec 1046 io.fast_uop.bits := RegNext(s1_out.uop) 1047 1048 // 1049 io.s2_ptr_chasing := RegEnable(s1_try_ptr_chasing && !s1_cancel_ptr_chasing, false.B, s1_fire) 1050 1051 io.prefetch_train.valid := s2_valid && !s2_actually_mmio && !s2_in.tlbMiss 1052 io.prefetch_train.bits.fromLsPipelineBundle(s2_in) 1053 io.prefetch_train.bits.miss := io.dcache.resp.bits.miss // TODO: use trace with bank conflict? 1054 io.prefetch_train.bits.meta_prefetch := io.dcache.resp.bits.meta_prefetch 1055 io.prefetch_train.bits.meta_access := io.dcache.resp.bits.meta_access 1056 1057 1058 io.prefetch_train_l1.valid := s2_valid && !s2_actually_mmio 1059 io.prefetch_train_l1.bits.fromLsPipelineBundle(s2_in) 1060 io.prefetch_train_l1.bits.miss := io.dcache.resp.bits.miss 1061 io.prefetch_train_l1.bits.meta_prefetch := io.dcache.resp.bits.meta_prefetch 1062 io.prefetch_train_l1.bits.meta_access := io.dcache.resp.bits.meta_access 1063 if (env.FPGAPlatform){ 1064 io.dcache.s0_pc := DontCare 1065 io.dcache.s1_pc := DontCare 1066 io.dcache.s2_pc := DontCare 1067 }else{ 1068 io.dcache.s0_pc := s0_out.uop.pc 1069 io.dcache.s1_pc := s1_out.uop.pc 1070 io.dcache.s2_pc := s2_out.uop.pc 1071 } 1072 io.dcache.s2_kill := s2_pmp.ld || s2_actually_mmio || s2_kill 1073 1074 val s1_ld_left_fire = s1_valid && !s1_kill && s2_ready 1075 val s2_ld_valid_dup = RegInit(0.U(6.W)) 1076 s2_ld_valid_dup := 0x0.U(6.W) 1077 when (s1_ld_left_fire && !s1_out.isHWPrefetch) { s2_ld_valid_dup := 0x3f.U(6.W) } 1078 when (s1_kill || s1_out.isHWPrefetch) { s2_ld_valid_dup := 0x0.U(6.W) } 1079 assert(RegNext((s2_valid === s2_ld_valid_dup(0)) || RegNext(s1_out.isHWPrefetch))) 1080 1081 // Pipeline 1082 // -------------------------------------------------------------------------------- 1083 // stage 3 1084 // -------------------------------------------------------------------------------- 1085 // writeback and update load queue 1086 val s3_valid = RegNext(s2_valid && !s2_out.isHWPrefetch && !s2_out.uop.robIdx.needFlush(io.redirect)) 1087 val s3_in = RegEnable(s2_out, s2_fire) 1088 val s3_out = Wire(Valid(new MemExuOutput)) 1089 val s3_dcache_rep = RegEnable(s2_dcache_fast_rep && s2_troublem, false.B, s2_fire) 1090 val s3_ld_valid_dup = RegEnable(s2_ld_valid_dup, s2_fire) 1091 val s3_fast_rep = Wire(Bool()) 1092 val s3_troublem = RegNext(s2_troublem) 1093 val s3_kill = s3_in.uop.robIdx.needFlush(io.redirect) 1094 val s3_vecout = Wire(new OnlyVecExuOutput) 1095 val s3_exp = RegEnable(s2_out.exp, true.B, s2_fire) 1096 val s3_isvec = RegEnable(s2_out.isvec, false.B, s2_fire) 1097 val s3_vec_alignedType = RegEnable(s2_vec_alignedType, s2_fire) 1098 s3_ready := !s3_valid || s3_kill || io.ldout.ready 1099 1100 // forwrad last beat 1101 val (s3_fwd_frm_d_chan, s3_fwd_data_frm_d_chan) = io.tl_d_channel.forward(s2_valid && s2_out.forward_tlDchannel, s2_out.mshrid, s2_out.paddr) 1102 val s3_fwd_data_valid = RegEnable(s2_fwd_data_valid, false.B, s2_valid) 1103 val s3_fwd_frm_d_chan_valid = (s3_fwd_frm_d_chan && s3_fwd_data_valid && s3_in.handledByMSHR) 1104 val s3_nuke = VecInit((0 until StorePipelineWidth).map(w => { 1105 io.stld_nuke_query(w).valid && // query valid 1106 isAfter(s3_in.uop.robIdx, io.stld_nuke_query(w).bits.robIdx) && // older store 1107 // TODO: Fix me when vector instruction 1108 (s3_in.paddr(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 1109 (s3_in.mask & io.stld_nuke_query(w).bits.mask).orR // data mask contain 1110 })).asUInt.orR && !s3_in.tlbMiss || s3_in.rep_info.nuke 1111 1112 1113 // s3 load fast replay 1114 io.fast_rep_out.valid := s3_valid && s3_fast_rep && !s3_in.uop.robIdx.needFlush(io.redirect) && !s3_isvec 1115 io.fast_rep_out.bits := s3_in 1116 1117 io.lsq.ldin.valid := s3_valid && (!s3_fast_rep || !io.fast_rep_out.ready) && !s3_in.feedbacked && !s3_in.lateKill && !s3_isvec 1118 io.lsq.ldin.bits := s3_in 1119 io.lsq.ldin.bits.miss := s3_in.miss && !s3_fwd_frm_d_chan_valid 1120 1121 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1122 io.lsq.ldin.bits.data_wen_dup := s3_ld_valid_dup.asBools 1123 io.lsq.ldin.bits.replacementUpdated := io.dcache.resp.bits.replacementUpdated 1124 io.lsq.ldin.bits.missDbUpdated := RegNext(s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated) 1125 1126 val s3_dly_ld_err = 1127 if (EnableAccurateLoadError) { 1128 (s3_in.lateKill || io.dcache.resp.bits.error_delayed) && RegNext(io.csrCtrl.cache_error_enable) 1129 } else { 1130 WireInit(false.B) 1131 } 1132 io.s3_dly_ld_err := false.B // s3_dly_ld_err && s3_valid 1133 io.fast_rep_out.bits.delayedLoadError := s3_dly_ld_err 1134 io.lsq.ldin.bits.dcacheRequireReplay := s3_dcache_rep 1135 1136 val s3_vp_match_fail = RegNext(io.lsq.forward.matchInvalid || io.sbuffer.matchInvalid) && s3_troublem 1137 val s3_ldld_rep_inst = 1138 io.lsq.ldld_nuke_query.resp.valid && 1139 io.lsq.ldld_nuke_query.resp.bits.rep_frm_fetch && 1140 RegNext(io.csrCtrl.ldld_vio_check_enable) 1141 1142 val s3_rep_info = WireInit(s3_in.rep_info) 1143 s3_rep_info.dcache_miss := s3_in.rep_info.dcache_miss && !s3_fwd_frm_d_chan_valid && s3_troublem 1144 val s3_rep_frm_fetch = s3_vp_match_fail || s3_ldld_rep_inst 1145 val s3_sel_rep_cause = PriorityEncoderOH(s3_rep_info.cause.asUInt) 1146 val s3_force_rep = s3_sel_rep_cause(LoadReplayCauses.C_TM) && 1147 !s3_in.uop.exceptionVec(loadAddrMisaligned) && 1148 s3_troublem 1149 1150 val s3_exception = ExceptionNO.selectByFu(s3_in.uop.exceptionVec, LduCfg).asUInt.orR && s3_exp 1151 when ((s3_exception || s3_dly_ld_err || s3_rep_frm_fetch) && !s3_force_rep) { 1152 io.lsq.ldin.bits.rep_info.cause := 0.U.asTypeOf(s3_rep_info.cause.cloneType) 1153 } .otherwise { 1154 io.lsq.ldin.bits.rep_info.cause := VecInit(s3_sel_rep_cause.asBools) 1155 } 1156 1157 // Int load, if hit, will be writebacked at s3 1158 s3_out.valid := s3_valid && !io.lsq.ldin.bits.rep_info.need_rep && !s3_in.mmio 1159 s3_out.bits.uop := s3_in.uop 1160 s3_out.bits.uop.exceptionVec(loadAccessFault) := (s3_dly_ld_err || s3_in.uop.exceptionVec(loadAccessFault)) && s3_exp 1161 s3_out.bits.uop.replayInst := s3_rep_frm_fetch 1162 s3_out.bits.data := s3_in.data 1163 s3_out.bits.debug.isMMIO := s3_in.mmio 1164 s3_out.bits.debug.isPerfCnt := false.B 1165 s3_out.bits.debug.paddr := s3_in.paddr 1166 s3_out.bits.debug.vaddr := s3_in.vaddr 1167 // Vector load 1168 s3_vecout.isvec := s3_isvec 1169 s3_vecout.vecdata := 0.U // Data will be assigned later 1170 s3_vecout.mask := s3_in.mask 1171 // s3_vecout.rob_idx_valid := s3_in.rob_idx_valid 1172 // s3_vecout.inner_idx := s3_in.inner_idx 1173 // s3_vecout.rob_idx := s3_in.rob_idx 1174 // s3_vecout.offset := s3_in.offset 1175 s3_vecout.reg_offset := s3_in.reg_offset 1176 s3_vecout.exp := s3_exp 1177 s3_vecout.is_first_ele := s3_in.is_first_ele 1178 s3_vecout.uopQueuePtr := DontCare // uopQueuePtr is already saved in flow queue 1179 s3_vecout.flowPtr := s3_in.flowPtr 1180 s3_vecout.elemIdx := DontCare // elemIdx is already saved in flow queue 1181 s3_vecout.elemIdxInsideVd := DontCare 1182 1183 when (s3_force_rep) { 1184 s3_out.bits.uop.exceptionVec := 0.U.asTypeOf(s3_in.uop.exceptionVec.cloneType) 1185 } 1186 1187 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1188 1189 io.lsq.ldin.bits.uop := s3_out.bits.uop 1190 1191 val s3_revoke = s3_exception || io.lsq.ldin.bits.rep_info.need_rep 1192 io.lsq.ldld_nuke_query.revoke := s3_revoke 1193 io.lsq.stld_nuke_query.revoke := s3_revoke 1194 1195 // feedback slow 1196 s3_fast_rep := RegNext(s2_fast_rep) && 1197 !s3_in.feedbacked && 1198 !s3_in.lateKill && 1199 !s3_rep_frm_fetch && 1200 !s3_exception 1201 1202 val s3_fb_no_waiting = !s3_in.isLoadReplay && !(s3_fast_rep && io.fast_rep_out.ready) && !s3_in.feedbacked 1203 1204 // 1205 io.feedback_slow.valid := s3_valid && !s3_in.uop.robIdx.needFlush(io.redirect) && s3_fb_no_waiting 1206 io.feedback_slow.bits.hit := !io.lsq.ldin.bits.rep_info.need_rep || io.lsq.ldin.ready 1207 io.feedback_slow.bits.flushState := s3_in.ptwBack 1208 io.feedback_slow.bits.robIdx := s3_in.uop.robIdx 1209 io.feedback_slow.bits.sourceType := RSFeedbackType.lrqFull 1210 io.feedback_slow.bits.dataInvalidSqIdx := DontCare 1211 1212 io.ldCancel.ld2Cancel.valid := s3_valid && s3_in.isFirstIssue && ( // issued from IQ 1213 io.lsq.ldin.bits.rep_info.need_rep || s3_in.mmio // exe fail or is mmio 1214 ) 1215 io.ldCancel.ld2Cancel.bits := s3_in.deqPortIdx 1216 1217 val s3_ld_wb_meta = Mux(s3_out.valid, s3_out.bits, io.lsq.uncache.bits) 1218 1219 // data from load queue refill 1220 val s3_ld_raw_data_frm_uncache = io.lsq.ld_raw_data 1221 val s3_merged_data_frm_uncache = s3_ld_raw_data_frm_uncache.mergedData() 1222 val s3_picked_data_frm_uncache = LookupTree(s3_ld_raw_data_frm_uncache.addrOffset, List( 1223 "b000".U -> s3_merged_data_frm_uncache(63, 0), 1224 "b001".U -> s3_merged_data_frm_uncache(63, 8), 1225 "b010".U -> s3_merged_data_frm_uncache(63, 16), 1226 "b011".U -> s3_merged_data_frm_uncache(63, 24), 1227 "b100".U -> s3_merged_data_frm_uncache(63, 32), 1228 "b101".U -> s3_merged_data_frm_uncache(63, 40), 1229 "b110".U -> s3_merged_data_frm_uncache(63, 48), 1230 "b111".U -> s3_merged_data_frm_uncache(63, 56) 1231 )) 1232 val s3_ld_data_frm_uncache = rdataHelper(s3_ld_raw_data_frm_uncache.uop, s3_picked_data_frm_uncache) 1233 1234 // data from dcache hit 1235 val s3_ld_raw_data_frm_cache = Wire(new LoadDataFromDcacheBundle) 1236 s3_ld_raw_data_frm_cache.respDcacheData := io.dcache.resp.bits.data_delayed 1237 s3_ld_raw_data_frm_cache.forwardMask := RegEnable(s2_fwd_mask, s2_valid) 1238 s3_ld_raw_data_frm_cache.forwardData := RegEnable(s2_fwd_data, s2_valid) 1239 s3_ld_raw_data_frm_cache.uop := RegEnable(s2_out.uop, s2_valid) 1240 s3_ld_raw_data_frm_cache.addrOffset := RegEnable(s2_out.paddr(3, 0), s2_valid) 1241 s3_ld_raw_data_frm_cache.forward_D := RegEnable(s2_fwd_frm_d_chan, false.B, s2_valid) || s3_fwd_frm_d_chan_valid 1242 s3_ld_raw_data_frm_cache.forwardData_D := Mux(s3_fwd_frm_d_chan_valid, s3_fwd_data_frm_d_chan, RegEnable(s2_fwd_data_frm_d_chan, s2_valid)) 1243 s3_ld_raw_data_frm_cache.forward_mshr := RegEnable(s2_fwd_frm_mshr, false.B, s2_valid) 1244 s3_ld_raw_data_frm_cache.forwardData_mshr := RegEnable(s2_fwd_data_frm_mshr, s2_valid) 1245 s3_ld_raw_data_frm_cache.forward_result_valid := RegEnable(s2_fwd_data_valid, false.B, s2_valid) 1246 1247 val s3_merged_data_frm_cache = s3_ld_raw_data_frm_cache.mergedData() 1248 val s3_picked_data_frm_cache = LookupTree(s3_ld_raw_data_frm_cache.addrOffset, List( 1249 "b0000".U -> s3_merged_data_frm_cache(63, 0), 1250 "b0001".U -> s3_merged_data_frm_cache(63, 8), 1251 "b0010".U -> s3_merged_data_frm_cache(63, 16), 1252 "b0011".U -> s3_merged_data_frm_cache(63, 24), 1253 "b0100".U -> s3_merged_data_frm_cache(63, 32), 1254 "b0101".U -> s3_merged_data_frm_cache(63, 40), 1255 "b0110".U -> s3_merged_data_frm_cache(63, 48), 1256 "b0111".U -> s3_merged_data_frm_cache(63, 56), 1257 "b1000".U -> s3_merged_data_frm_cache(127, 64), 1258 "b1001".U -> s3_merged_data_frm_cache(127, 72), 1259 "b1010".U -> s3_merged_data_frm_cache(127, 80), 1260 "b1011".U -> s3_merged_data_frm_cache(127, 88), 1261 "b1100".U -> s3_merged_data_frm_cache(127, 96), 1262 "b1101".U -> s3_merged_data_frm_cache(127, 104), 1263 "b1110".U -> s3_merged_data_frm_cache(127, 112), 1264 "b1111".U -> s3_merged_data_frm_cache(127, 120) 1265 )) 1266 val s3_ld_data_frm_cache = rdataHelper(s3_ld_raw_data_frm_cache.uop, s3_picked_data_frm_cache) 1267 1268 // FIXME: add 1 cycle delay ? 1269 io.lsq.uncache.ready := !s3_out.valid 1270 io.ldout.bits := s3_ld_wb_meta 1271 io.ldout.bits.data := Mux(s3_out.valid, s3_ld_data_frm_cache, s3_ld_data_frm_uncache) 1272 io.ldout.valid := !s3_vecout.isvec && 1273 (s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) || 1274 io.lsq.uncache.valid && !io.lsq.uncache.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid) 1275 1276 // vector output 1277 io.vecldout.bits.vec := s3_vecout 1278 // TODO: VLSU, uncache data logic 1279 val vecdata = rdataVecHelper(s3_vec_alignedType, s3_picked_data_frm_cache) 1280 io.vecldout.bits.vec.vecdata := vecdata 1281 io.vecldout.bits.data := 0.U 1282 // io.vecldout.bits.fflags := s3_out.bits.fflags 1283 // io.vecldout.bits.redirectValid := s3_out.bits.redirectValid 1284 // io.vecldout.bits.redirect := s3_out.bits.redirect 1285 io.vecldout.bits.debug := s3_out.bits.debug 1286 io.vecldout.bits.uop := s3_out.bits.uop 1287 io.vecldout.valid := s3_vecout.isvec && 1288 (s3_valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) || 1289 io.lsq.uncache.valid && !io.lsq.uncache.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid) && 1290 !io.lsq.ldin.bits.rep_info.need_rep 1291 1292 io.vecReplay.valid := s3_vecout.isvec && s3_valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) && 1293 io.lsq.ldin.bits.rep_info.need_rep 1294 io.vecReplay.bits := DontCare 1295 io.vecReplay.bits.uop := s3_in.uop 1296 io.vecReplay.bits.vaddr := s3_in.vaddr 1297 io.vecReplay.bits.paddr := s3_in.paddr 1298 io.vecReplay.bits.mask := s3_in.mask 1299 io.vecReplay.bits.isvec := true.B 1300 io.vecReplay.bits.uop_unit_stride_fof := s3_in.uop_unit_stride_fof 1301 io.vecReplay.bits.reg_offset := s3_in.reg_offset 1302 io.vecReplay.bits.exp := s3_in.exp 1303 io.vecReplay.bits.is_first_ele := s3_in.is_first_ele 1304 io.vecReplay.bits.flowPtr := s3_in.flowPtr 1305 1306 // fast load to load forward 1307 io.l2l_fwd_out.valid := s3_out.valid && !s3_in.lateKill 1308 io.l2l_fwd_out.data := s3_ld_data_frm_cache 1309 io.l2l_fwd_out.dly_ld_err := s3_dly_ld_err // ecc delayed error 1310 1311 // trigger 1312 val last_valid_data = RegNext(RegEnable(io.ldout.bits.data, io.ldout.fire)) 1313 val hit_ld_addr_trig_hit_vec = Wire(Vec(TriggerNum, Bool())) 1314 val lq_ld_addr_trig_hit_vec = io.lsq.trigger.lqLoadAddrTriggerHitVec 1315 (0 until TriggerNum).map{i => { 1316 val tdata2 = RegNext(io.trigger(i).tdata2) 1317 val matchType = RegNext(io.trigger(i).matchType) 1318 val tEnable = RegNext(io.trigger(i).tEnable) 1319 1320 hit_ld_addr_trig_hit_vec(i) := TriggerCmp(RegNext(s2_out.vaddr), tdata2, matchType, tEnable) 1321 io.trigger(i).addrHit := Mux(s3_out.valid, hit_ld_addr_trig_hit_vec(i), lq_ld_addr_trig_hit_vec(i)) 1322 io.trigger(i).lastDataHit := TriggerCmp(last_valid_data, tdata2, matchType, tEnable) 1323 }} 1324 io.lsq.trigger.hitLoadAddrTriggerHitVec := hit_ld_addr_trig_hit_vec 1325 1326 // FIXME: please move this part to LoadQueueReplay 1327 io.debug_ls := DontCare 1328 1329 // Topdown 1330 io.lsTopdownInfo.s1.robIdx := s1_in.uop.robIdx.value 1331 io.lsTopdownInfo.s1.vaddr_valid := s1_valid && s1_in.hasROBEntry 1332 io.lsTopdownInfo.s1.vaddr_bits := s1_vaddr 1333 io.lsTopdownInfo.s2.robIdx := s2_in.uop.robIdx.value 1334 io.lsTopdownInfo.s2.paddr_valid := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss 1335 io.lsTopdownInfo.s2.paddr_bits := s2_in.paddr 1336 io.lsTopdownInfo.s2.first_real_miss := io.dcache.resp.bits.real_miss 1337 io.lsTopdownInfo.s2.cache_miss_en := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated 1338 1339 // perf cnt 1340 XSPerfAccumulate("s0_in_valid", io.ldin.valid) 1341 XSPerfAccumulate("s0_in_block", io.ldin.valid && !io.ldin.fire) 1342 XSPerfAccumulate("s0_in_fire_first_issue", s0_valid && s0_isFirstIssue) 1343 XSPerfAccumulate("s0_lsq_fire_first_issue", io.replay.fire) 1344 XSPerfAccumulate("s0_ldu_fire_first_issue", io.ldin.fire && s0_isFirstIssue) 1345 XSPerfAccumulate("s0_fast_replay_issue", io.fast_rep_in.fire) 1346 XSPerfAccumulate("s0_stall_out", s0_valid && !s0_can_go) 1347 XSPerfAccumulate("s0_stall_dcache", s0_valid && !io.dcache.req.ready) 1348 XSPerfAccumulate("s0_addr_spec_success", s0_fire && s0_vaddr(VAddrBits-1, 12) === io.ldin.bits.src(0)(VAddrBits-1, 12)) 1349 XSPerfAccumulate("s0_addr_spec_failed", s0_fire && s0_vaddr(VAddrBits-1, 12) =/= io.ldin.bits.src(0)(VAddrBits-1, 12)) 1350 XSPerfAccumulate("s0_addr_spec_success_once", s0_fire && s0_vaddr(VAddrBits-1, 12) === io.ldin.bits.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 1351 XSPerfAccumulate("s0_addr_spec_failed_once", s0_fire && s0_vaddr(VAddrBits-1, 12) =/= io.ldin.bits.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 1352 XSPerfAccumulate("s0_forward_tl_d_channel", s0_out.forward_tlDchannel) 1353 XSPerfAccumulate("s0_hardware_prefetch_fire", s0_fire && s0_hw_prf_select) 1354 XSPerfAccumulate("s0_software_prefetch_fire", s0_fire && s0_prf && s0_int_iss_select) 1355 XSPerfAccumulate("s0_hardware_prefetch_blocked", io.prefetch_req.valid && !s0_hw_prf_select) 1356 XSPerfAccumulate("s0_hardware_prefetch_total", io.prefetch_req.valid) 1357 1358 XSPerfAccumulate("s1_in_valid", s1_valid) 1359 XSPerfAccumulate("s1_in_fire", s1_fire) 1360 XSPerfAccumulate("s1_in_fire_first_issue", s1_fire && s1_in.isFirstIssue) 1361 XSPerfAccumulate("s1_tlb_miss", s1_fire && s1_tlb_miss) 1362 XSPerfAccumulate("s1_tlb_miss_first_issue", s1_fire && s1_tlb_miss && s1_in.isFirstIssue) 1363 XSPerfAccumulate("s1_stall_out", s1_valid && !s1_can_go) 1364 XSPerfAccumulate("s1_late_kill", s1_valid && s1_fast_rep_kill) 1365 1366 XSPerfAccumulate("s2_in_valid", s2_valid) 1367 XSPerfAccumulate("s2_in_fire", s2_fire) 1368 XSPerfAccumulate("s2_in_fire_first_issue", s2_fire && s2_in.isFirstIssue) 1369 XSPerfAccumulate("s2_dcache_miss", s2_fire && io.dcache.resp.bits.miss) 1370 XSPerfAccumulate("s2_dcache_miss_first_issue", s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1371 XSPerfAccumulate("s2_dcache_real_miss_first_issue", s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1372 XSPerfAccumulate("s2_full_forward", s2_fire && s2_full_fwd) 1373 XSPerfAccumulate("s2_dcache_miss_full_forward", s2_fire && s2_dcache_miss) 1374 XSPerfAccumulate("s2_fwd_frm_d_can", s2_valid && s2_fwd_frm_d_chan) 1375 XSPerfAccumulate("s2_fwd_frm_d_chan_or_mshr", s2_valid && s2_fwd_frm_d_chan_or_mshr) 1376 XSPerfAccumulate("s2_stall_out", s2_fire && !s2_can_go) 1377 XSPerfAccumulate("s2_prefetch", s2_fire && s2_prf) 1378 XSPerfAccumulate("s2_prefetch_ignored", s2_fire && s2_prf && s2_mq_nack) // ignore prefetch for mshr full / miss req port conflict 1379 XSPerfAccumulate("s2_prefetch_miss", s2_fire && s2_prf && io.dcache.resp.bits.miss) // prefetch req miss in l1 1380 XSPerfAccumulate("s2_prefetch_hit", s2_fire && s2_prf && !io.dcache.resp.bits.miss) // prefetch req hit in l1 1381 XSPerfAccumulate("s2_prefetch_accept", s2_fire && s2_prf && io.dcache.resp.bits.miss && !s2_mq_nack) // prefetch a missed line in l1, and l1 accepted it 1382 XSPerfAccumulate("s2_forward_req", s2_fire && s2_in.forward_tlDchannel) 1383 XSPerfAccumulate("s2_successfully_forward_channel_D", s2_fire && s2_fwd_frm_d_chan && s2_fwd_data_valid) 1384 XSPerfAccumulate("s2_successfully_forward_mshr", s2_fire && s2_fwd_frm_mshr && s2_fwd_data_valid) 1385 1386 XSPerfAccumulate("s3_fwd_frm_d_chan", s3_valid && s3_fwd_frm_d_chan_valid) 1387 1388 XSPerfAccumulate("load_to_load_forward", s1_try_ptr_chasing && !s1_ptr_chasing_canceled) 1389 XSPerfAccumulate("load_to_load_forward_try", s1_try_ptr_chasing) 1390 XSPerfAccumulate("load_to_load_forward_fail", s1_cancel_ptr_chasing) 1391 XSPerfAccumulate("load_to_load_forward_fail_cancelled", s1_cancel_ptr_chasing && s1_ptr_chasing_canceled) 1392 XSPerfAccumulate("load_to_load_forward_fail_wakeup_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && s1_not_fast_match) 1393 XSPerfAccumulate("load_to_load_forward_fail_op_not_ld", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && s1_fu_op_type_not_ld) 1394 XSPerfAccumulate("load_to_load_forward_fail_addr_align", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && s1_addr_misaligned) 1395 XSPerfAccumulate("load_to_load_forward_fail_set_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && !s1_addr_misaligned && s1_addr_mismatch) 1396 1397 // bug lyq: some signals in perfEvents are no longer suitable for the current MemBlock design 1398 // hardware performance counter 1399 val perfEvents = Seq( 1400 ("load_s0_in_fire ", s0_fire ), 1401 ("load_to_load_forward ", s1_fire && s1_try_ptr_chasing && !s1_ptr_chasing_canceled ), 1402 ("stall_dcache ", s0_valid && s0_can_go && !io.dcache.req.ready ), 1403 ("load_s1_in_fire ", s0_fire ), 1404 ("load_s1_tlb_miss ", s1_fire && io.tlb.resp.bits.miss ), 1405 ("load_s2_in_fire ", s1_fire ), 1406 ("load_s2_dcache_miss ", s2_fire && io.dcache.resp.bits.miss ), 1407 ) 1408 generatePerfEvent() 1409 1410 when(io.ldout.fire){ 1411 XSDebug("ldout %x\n", io.ldout.bits.uop.pc) 1412 } 1413 // end 1414}