1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.fu.PMPRespBundle 27import xiangshan.backend.rob.{DebugLsInfoBundle, LsTopdownInfo, RobPtr} 28import xiangshan.cache._ 29import xiangshan.cache.wpu.ReplayCarry 30import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp} 31import xiangshan.mem.mdp._ 32 33class LoadToLsqReplayIO(implicit p: Parameters) extends XSBundle with HasDCacheParameters { 34 // mshr refill index 35 val mshr_id = UInt(log2Up(cfg.nMissEntries).W) 36 // get full data from store queue and sbuffer 37 val full_fwd = Bool() 38 // wait for data from store inst's store queue index 39 val data_inv_sq_idx = new SqPtr 40 // wait for address from store queue index 41 val addr_inv_sq_idx = new SqPtr 42 // replay carry 43 val rep_carry = new ReplayCarry(nWays) 44 // data in last beat 45 val last_beat = Bool() 46 // replay cause 47 val cause = Vec(LoadReplayCauses.allCauses, Bool()) 48 // performance debug information 49 val debug = new PerfDebugInfo 50 51 // alias 52 def mem_amb = cause(LoadReplayCauses.C_MA) 53 def tlb_miss = cause(LoadReplayCauses.C_TM) 54 def fwd_fail = cause(LoadReplayCauses.C_FF) 55 def dcache_rep = cause(LoadReplayCauses.C_DR) 56 def dcache_miss = cause(LoadReplayCauses.C_DM) 57 def wpu_fail = cause(LoadReplayCauses.C_WF) 58 def bank_conflict = cause(LoadReplayCauses.C_BC) 59 def rar_nack = cause(LoadReplayCauses.C_RAR) 60 def raw_nack = cause(LoadReplayCauses.C_RAW) 61 def nuke = cause(LoadReplayCauses.C_NK) 62 def need_rep = cause.asUInt.orR 63} 64 65 66class LoadToLsqIO(implicit p: Parameters) extends XSBundle { 67 val ldin = DecoupledIO(new LqWriteBundle) 68 val uncache = Flipped(DecoupledIO(new ExuOutput)) 69 val ld_raw_data = Input(new LoadDataFromLQBundle) 70 val forward = new PipeLoadForwardQueryIO 71 val stld_nuke_query = new LoadNukeQueryIO 72 val ldld_nuke_query = new LoadNukeQueryIO 73 val trigger = Flipped(new LqTriggerIO) 74} 75 76class LoadToLoadIO(implicit p: Parameters) extends XSBundle { 77 val valid = Bool() 78 val data = UInt(XLEN.W) // load to load fast path is limited to ld (64 bit) used as vaddr src1 only 79 val dly_ld_err = Bool() 80} 81 82class LoadUnitTriggerIO(implicit p: Parameters) extends XSBundle { 83 val tdata2 = Input(UInt(64.W)) 84 val matchType = Input(UInt(2.W)) 85 val tEnable = Input(Bool()) // timing is calculated before this 86 val addrHit = Output(Bool()) 87 val lastDataHit = Output(Bool()) 88} 89 90class LoadUnit(implicit p: Parameters) extends XSModule 91 with HasLoadHelper 92 with HasPerfEvents 93 with HasDCacheParameters 94 with HasCircularQueuePtrHelper 95{ 96 val io = IO(new Bundle() { 97 // control 98 val redirect = Flipped(ValidIO(new Redirect)) 99 val csrCtrl = Flipped(new CustomCSRCtrlIO) 100 101 // int issue path 102 val ldin = Flipped(Decoupled(new ExuInput)) 103 val ldout = Decoupled(new ExuOutput) 104 val rsIdx = Input(UInt()) 105 val isFirstIssue = Input(Bool()) 106 107 // data path 108 val tlb = new TlbRequestIO(2) 109 val pmp = Flipped(new PMPRespBundle()) // arrive same to tlb now 110 val dcache = new DCacheLoadIO 111 val sbuffer = new LoadForwardQueryIO 112 val lsq = new LoadToLsqIO 113 val tl_d_channel = Input(new DcacheToLduForwardIO) 114 val forward_mshr = Flipped(new LduToMissqueueForwardIO) 115 val refill = Flipped(ValidIO(new Refill)) 116 val l2_hint = Input(Valid(new L2ToL1Hint)) 117 118 // fast wakeup 119 val fast_uop = ValidIO(new MicroOp) // early wakeup signal generated in load_s1, send to RS in load_s2 120 121 // trigger 122 val trigger = Vec(3, new LoadUnitTriggerIO) 123 124 // prefetch 125 val prefetch_train = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to sms 126 val prefetch_train_l1 = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to stream & stride 127 val prefetch_req = Flipped(ValidIO(new L1PrefetchReq)) // hardware prefetch to l1 cache req 128 val canAcceptLowConfPrefetch = Output(Bool()) 129 val canAcceptHighConfPrefetch = Output(Bool()) 130 131 // load to load fast path 132 val l2l_fwd_in = Input(new LoadToLoadIO) 133 val l2l_fwd_out = Output(new LoadToLoadIO) 134 135 val ld_fast_match = Input(Bool()) 136 val ld_fast_fuOpType = Input(UInt()) 137 val ld_fast_imm = Input(UInt(12.W)) 138 139 // rs feedback 140 val feedback_fast = ValidIO(new RSFeedback) // stage 2 141 val feedback_slow = ValidIO(new RSFeedback) // stage 3 142 143 // load ecc error 144 val s3_dly_ld_err = Output(Bool()) // Note that io.s3_dly_ld_err and io.lsq.s3_dly_ld_err is different 145 146 // schedule error query 147 val stld_nuke_query = Flipped(Vec(StorePipelineWidth, Valid(new StoreNukeQueryIO))) 148 149 // queue-based replay 150 val replay = Flipped(Decoupled(new LsPipelineBundle)) 151 val lq_rep_full = Input(Bool()) 152 153 // misc 154 val s2_ptr_chasing = Output(Bool()) // provide right pc for hw prefetch 155 156 // Load fast replay path 157 val fast_rep_in = Flipped(Decoupled(new LqWriteBundle)) 158 val fast_rep_out = Decoupled(new LqWriteBundle) 159 160 // perf 161 val debug_ls = Output(new DebugLsInfoBundle) 162 val lsTopdownInfo = Output(new LsTopdownInfo) 163 val correctMissTrain = Input(Bool()) 164 }) 165 166 val s1_ready, s2_ready, s3_ready = WireInit(false.B) 167 168 // Pipeline 169 // -------------------------------------------------------------------------------- 170 // stage 0 171 // -------------------------------------------------------------------------------- 172 // generate addr, use addr to query DCache and DTLB 173 val s0_valid = Wire(Bool()) 174 val s0_kill = Wire(Bool()) 175 val s0_vaddr = Wire(UInt(VAddrBits.W)) 176 val s0_mask = Wire(UInt((VLEN/8).W)) 177 val s0_uop = Wire(new MicroOp) 178 val s0_has_rob_entry = Wire(Bool()) 179 val s0_rsIdx = Wire(UInt(log2Up(IssQueSize).W)) 180 val s0_mshrid = Wire(UInt()) 181 val s0_try_l2l = Wire(Bool()) 182 val s0_rep_carry = Wire(new ReplayCarry(nWays)) 183 val s0_isFirstIssue = Wire(Bool()) 184 val s0_fast_rep = Wire(Bool()) 185 val s0_ld_rep = Wire(Bool()) 186 val s0_l2l_fwd = Wire(Bool()) 187 val s0_sched_idx = Wire(UInt()) 188 val s0_can_go = s1_ready 189 val s0_fire = s0_valid && s0_can_go 190 val s0_out = Wire(new LqWriteBundle) 191 192 // load flow select/gen 193 // src0: super load replayed by LSQ (cache miss replay) (io.replay) 194 // src1: fast load replay (io.fast_rep_in) 195 // src2: load replayed by LSQ (io.replay) 196 // src3: hardware prefetch from prefetchor (high confidence) (io.prefetch) 197 // src4: int read / software prefetch first issue from RS (io.in) 198 // src5: vec read first issue from RS (TODO) 199 // src6: load try pointchaising when no issued or replayed load (io.fastpath) 200 // src7: hardware prefetch from prefetchor (high confidence) (io.prefetch) 201 // priority: high to low 202 val s0_rep_stall = io.ldin.valid && isAfter(io.replay.bits.uop.robIdx, io.ldin.bits.uop.robIdx) 203 val s0_super_ld_rep_valid = io.replay.valid && io.replay.bits.forward_tlDchannel 204 val s0_ld_fast_rep_valid = io.fast_rep_in.valid 205 val s0_ld_rep_valid = io.replay.valid && !io.replay.bits.forward_tlDchannel && !s0_rep_stall 206 val s0_high_conf_prf_valid = io.prefetch_req.valid && io.prefetch_req.bits.confidence > 0.U 207 val s0_int_iss_valid = io.ldin.valid // int flow first issue or software prefetch 208 val s0_vec_iss_valid = WireInit(false.B) // TODO 209 val s0_l2l_fwd_valid = io.l2l_fwd_in.valid && io.ld_fast_match 210 val s0_low_conf_prf_valid = io.prefetch_req.valid && io.prefetch_req.bits.confidence === 0.U 211 dontTouch(s0_super_ld_rep_valid) 212 dontTouch(s0_ld_fast_rep_valid) 213 dontTouch(s0_ld_rep_valid) 214 dontTouch(s0_high_conf_prf_valid) 215 dontTouch(s0_int_iss_valid) 216 dontTouch(s0_vec_iss_valid) 217 dontTouch(s0_l2l_fwd_valid) 218 dontTouch(s0_low_conf_prf_valid) 219 220 // load flow source ready 221 val s0_super_ld_rep_ready = WireInit(true.B) 222 val s0_ld_fast_rep_ready = !s0_super_ld_rep_valid 223 val s0_ld_rep_ready = !s0_super_ld_rep_valid && 224 !s0_ld_fast_rep_valid 225 val s0_high_conf_prf_ready = !s0_super_ld_rep_valid && 226 !s0_ld_fast_rep_valid && 227 !s0_ld_rep_valid 228 229 val s0_int_iss_ready = !s0_super_ld_rep_valid && 230 !s0_ld_fast_rep_valid && 231 !s0_ld_rep_valid && 232 !s0_high_conf_prf_valid 233 234 val s0_vec_iss_ready = !s0_super_ld_rep_valid && 235 !s0_ld_fast_rep_valid && 236 !s0_ld_rep_valid && 237 !s0_high_conf_prf_valid && 238 !s0_int_iss_valid 239 240 val s0_l2l_fwd_ready = !s0_super_ld_rep_valid && 241 !s0_ld_fast_rep_valid && 242 !s0_ld_rep_valid && 243 !s0_high_conf_prf_valid && 244 !s0_int_iss_valid && 245 !s0_vec_iss_valid 246 247 val s0_low_conf_prf_ready = !s0_super_ld_rep_valid && 248 !s0_ld_fast_rep_valid && 249 !s0_ld_rep_valid && 250 !s0_high_conf_prf_valid && 251 !s0_int_iss_valid && 252 !s0_vec_iss_valid && 253 !s0_l2l_fwd_valid 254 dontTouch(s0_super_ld_rep_ready) 255 dontTouch(s0_ld_fast_rep_ready) 256 dontTouch(s0_ld_rep_ready) 257 dontTouch(s0_high_conf_prf_ready) 258 dontTouch(s0_int_iss_ready) 259 dontTouch(s0_vec_iss_ready) 260 dontTouch(s0_l2l_fwd_ready) 261 dontTouch(s0_low_conf_prf_ready) 262 263 // load flow source select (OH) 264 val s0_super_ld_rep_select = s0_super_ld_rep_valid && s0_super_ld_rep_ready 265 val s0_ld_fast_rep_select = s0_ld_fast_rep_valid && s0_ld_fast_rep_ready 266 val s0_ld_rep_select = s0_ld_rep_valid && s0_ld_rep_ready 267 val s0_hw_prf_select = s0_high_conf_prf_ready && s0_high_conf_prf_valid || 268 s0_low_conf_prf_ready && s0_low_conf_prf_valid 269 val s0_int_iss_select = s0_int_iss_ready && s0_int_iss_valid 270 val s0_vec_iss_select = s0_vec_iss_ready && s0_vec_iss_valid 271 val s0_l2l_fwd_select = s0_l2l_fwd_ready && s0_l2l_fwd_valid 272 assert(!s0_vec_iss_select) // to be added 273 dontTouch(s0_super_ld_rep_select) 274 dontTouch(s0_ld_fast_rep_select) 275 dontTouch(s0_ld_rep_select) 276 dontTouch(s0_hw_prf_select) 277 dontTouch(s0_int_iss_select) 278 dontTouch(s0_vec_iss_select) 279 dontTouch(s0_l2l_fwd_select) 280 281 s0_valid := (s0_super_ld_rep_valid || 282 s0_ld_fast_rep_valid || 283 s0_ld_rep_valid || 284 s0_high_conf_prf_valid || 285 s0_int_iss_valid || 286 s0_vec_iss_valid || 287 s0_l2l_fwd_valid || 288 s0_low_conf_prf_valid) && io.dcache.req.ready && !s0_kill 289 290 // which is S0's out is ready and dcache is ready 291 val s0_try_ptr_chasing = s0_l2l_fwd_select 292 val s0_do_try_ptr_chasing = s0_try_ptr_chasing && s0_can_go && io.dcache.req.ready 293 val s0_ptr_chasing_vaddr = io.l2l_fwd_in.data(5, 0) +& io.ld_fast_imm(5, 0) 294 val s0_ptr_chasing_canceled = WireInit(false.B) 295 s0_kill := s0_ptr_chasing_canceled || (s0_out.uop.robIdx.needFlush(io.redirect) && !s0_try_ptr_chasing) 296 297 // prefetch related ctrl signal 298 val s0_prf = Wire(Bool()) 299 val s0_prf_rd = Wire(Bool()) 300 val s0_prf_wr = Wire(Bool()) 301 val s0_hw_prf = s0_hw_prf_select 302 303 io.canAcceptLowConfPrefetch := s0_low_conf_prf_ready 304 io.canAcceptHighConfPrefetch := s0_high_conf_prf_ready 305 306 // query DTLB 307 io.tlb.req.valid := s0_valid 308 io.tlb.req.bits.cmd := Mux(s0_prf, 309 Mux(s0_prf_wr, TlbCmd.write, TlbCmd.read), 310 TlbCmd.read 311 ) 312 io.tlb.req.bits.vaddr := Mux(s0_hw_prf_select, io.prefetch_req.bits.paddr, s0_vaddr) 313 io.tlb.req.bits.size := LSUOpType.size(s0_uop.ctrl.fuOpType) 314 io.tlb.req.bits.kill := s0_kill 315 io.tlb.req.bits.memidx.is_ld := true.B 316 io.tlb.req.bits.memidx.is_st := false.B 317 io.tlb.req.bits.memidx.idx := s0_uop.lqIdx.value 318 io.tlb.req.bits.debug.robIdx := s0_uop.robIdx 319 io.tlb.req.bits.no_translate := s0_hw_prf_select // hw b.reqetch addr does not need to be translated 320 io.tlb.req.bits.debug.pc := s0_uop.cf.pc 321 io.tlb.req.bits.debug.isFirstIssue := s0_isFirstIssue 322 323 // query DCache 324 io.dcache.req.valid := s0_valid 325 io.dcache.req.bits.cmd := Mux(s0_prf_rd, 326 MemoryOpConstants.M_PFR, 327 Mux(s0_prf_wr, MemoryOpConstants.M_PFW, MemoryOpConstants.M_XRD) 328 ) 329 io.dcache.req.bits.vaddr := s0_vaddr 330 io.dcache.req.bits.mask := s0_mask 331 io.dcache.req.bits.data := DontCare 332 io.dcache.req.bits.isFirstIssue := s0_isFirstIssue 333 io.dcache.req.bits.instrtype := Mux(s0_prf, DCACHE_PREFETCH_SOURCE.U, LOAD_SOURCE.U) 334 io.dcache.req.bits.debug_robIdx := s0_uop.robIdx.value 335 io.dcache.req.bits.replayCarry := s0_rep_carry 336 io.dcache.req.bits.id := DontCare // TODO: update cache meta 337 io.dcache.pf_source := Mux(s0_hw_prf_select, io.prefetch_req.bits.pf_source.value, L1_HW_PREFETCH_NULL) 338 339 // load flow priority mux 340 def fromNullSource() = { 341 s0_vaddr := 0.U 342 s0_mask := 0.U 343 s0_uop := 0.U.asTypeOf(new MicroOp) 344 s0_try_l2l := false.B 345 s0_has_rob_entry := false.B 346 s0_rsIdx := 0.U 347 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 348 s0_mshrid := 0.U 349 s0_isFirstIssue := false.B 350 s0_fast_rep := false.B 351 s0_ld_rep := false.B 352 s0_l2l_fwd := false.B 353 s0_prf := false.B 354 s0_prf_rd := false.B 355 s0_prf_wr := false.B 356 s0_sched_idx := 0.U 357 } 358 359 def fromFastReplaySource(src: LqWriteBundle) = { 360 s0_vaddr := src.vaddr 361 s0_mask := src.mask 362 s0_uop := src.uop 363 s0_try_l2l := false.B 364 s0_has_rob_entry := src.hasROBEntry 365 s0_rep_carry := src.rep_info.rep_carry 366 s0_mshrid := src.rep_info.mshr_id 367 s0_rsIdx := src.rsIdx 368 s0_isFirstIssue := false.B 369 s0_fast_rep := true.B 370 s0_ld_rep := src.isLoadReplay 371 s0_l2l_fwd := false.B 372 s0_prf := LSUOpType.isPrefetch(src.uop.ctrl.fuOpType) 373 s0_prf_rd := src.uop.ctrl.fuOpType === LSUOpType.prefetch_r 374 s0_prf_wr := src.uop.ctrl.fuOpType === LSUOpType.prefetch_w 375 s0_sched_idx := src.schedIndex 376 } 377 378 def fromNormalReplaySource(src: LsPipelineBundle) = { 379 s0_vaddr := src.vaddr 380 s0_mask := genVWmask(src.vaddr, src.uop.ctrl.fuOpType(1, 0)) 381 s0_uop := src.uop 382 s0_try_l2l := false.B 383 s0_has_rob_entry := true.B 384 s0_rsIdx := src.rsIdx 385 s0_rep_carry := src.replayCarry 386 s0_mshrid := src.mshrid 387 s0_isFirstIssue := false.B 388 s0_fast_rep := false.B 389 s0_ld_rep := true.B 390 s0_l2l_fwd := false.B 391 s0_prf := LSUOpType.isPrefetch(src.uop.ctrl.fuOpType) 392 s0_prf_rd := src.uop.ctrl.fuOpType === LSUOpType.prefetch_r 393 s0_prf_wr := src.uop.ctrl.fuOpType === LSUOpType.prefetch_w 394 s0_sched_idx := src.schedIndex 395 } 396 397 def fromPrefetchSource(src: L1PrefetchReq) = { 398 s0_vaddr := src.getVaddr() 399 s0_mask := 0.U 400 s0_uop := DontCare 401 s0_try_l2l := false.B 402 s0_has_rob_entry := false.B 403 s0_rsIdx := 0.U 404 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 405 s0_mshrid := 0.U 406 s0_isFirstIssue := false.B 407 s0_fast_rep := false.B 408 s0_ld_rep := false.B 409 s0_l2l_fwd := false.B 410 s0_prf := true.B 411 s0_prf_rd := !src.is_store 412 s0_prf_wr := src.is_store 413 s0_sched_idx := 0.U 414 } 415 416 def fromIntIssueSource(src: ExuInput) = { 417 s0_vaddr := src.src(0) + SignExt(src.uop.ctrl.imm(11, 0), VAddrBits) 418 s0_mask := genVWmask(s0_vaddr, src.uop.ctrl.fuOpType(1,0)) 419 s0_uop := src.uop 420 s0_try_l2l := false.B 421 s0_has_rob_entry := true.B 422 s0_rsIdx := io.rsIdx 423 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 424 s0_mshrid := 0.U 425 s0_isFirstIssue := true.B 426 s0_fast_rep := false.B 427 s0_ld_rep := false.B 428 s0_l2l_fwd := false.B 429 s0_prf := LSUOpType.isPrefetch(src.uop.ctrl.fuOpType) 430 s0_prf_rd := src.uop.ctrl.fuOpType === LSUOpType.prefetch_r 431 s0_prf_wr := src.uop.ctrl.fuOpType === LSUOpType.prefetch_w 432 s0_sched_idx := 0.U 433 } 434 435 def fromVecIssueSource() = { 436 s0_vaddr := 0.U 437 s0_mask := 0.U 438 s0_uop := 0.U.asTypeOf(new MicroOp) 439 s0_try_l2l := false.B 440 s0_has_rob_entry := false.B 441 s0_rsIdx := 0.U 442 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 443 s0_mshrid := 0.U 444 s0_isFirstIssue := false.B 445 s0_fast_rep := false.B 446 s0_ld_rep := false.B 447 s0_l2l_fwd := false.B 448 s0_prf := false.B 449 s0_prf_rd := false.B 450 s0_prf_wr := false.B 451 s0_sched_idx := 0.U 452 } 453 454 def fromLoadToLoadSource(src: LoadToLoadIO) = { 455 s0_vaddr := Cat(src.data(XLEN-1, 6), s0_ptr_chasing_vaddr(5,0)) 456 s0_mask := genVWmask(s0_vaddr, io.ld_fast_fuOpType(1, 0)) 457 // When there's no valid instruction from RS and LSQ, we try the load-to-load forwarding. 458 // Assume the pointer chasing is always ld. 459 s0_uop.ctrl.fuOpType := io.ld_fast_fuOpType 460 s0_try_l2l := true.B 461 // we dont care s0_isFirstIssue and s0_rsIdx and s0_sqIdx in S0 when trying pointchasing 462 // because these signals will be updated in S1 463 s0_has_rob_entry := false.B 464 s0_rsIdx := 0.U 465 s0_mshrid := 0.U 466 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 467 s0_isFirstIssue := true.B 468 s0_fast_rep := false.B 469 s0_ld_rep := false.B 470 s0_l2l_fwd := true.B 471 s0_prf := false.B 472 s0_prf_rd := false.B 473 s0_prf_wr := false.B 474 s0_sched_idx := 0.U 475 } 476 477 // set default 478 s0_uop := DontCare 479 when (s0_super_ld_rep_select) { fromNormalReplaySource(io.replay.bits) } 480 .elsewhen (s0_ld_fast_rep_select) { fromFastReplaySource(io.fast_rep_in.bits) } 481 .elsewhen (s0_ld_rep_select) { fromNormalReplaySource(io.replay.bits) } 482 .elsewhen (s0_hw_prf_select) { fromPrefetchSource(io.prefetch_req.bits) } 483 .elsewhen (s0_int_iss_select) { fromIntIssueSource(io.ldin.bits) } 484 .elsewhen (s0_vec_iss_select) { fromVecIssueSource() } 485 .otherwise { 486 if (EnableLoadToLoadForward) { 487 fromLoadToLoadSource(io.l2l_fwd_in) 488 } else { 489 fromNullSource() 490 } 491 } 492 493 // address align check 494 val s0_addr_aligned = LookupTree(s0_uop.ctrl.fuOpType(1, 0), List( 495 "b00".U -> true.B, //b 496 "b01".U -> (s0_vaddr(0) === 0.U), //h 497 "b10".U -> (s0_vaddr(1, 0) === 0.U), //w 498 "b11".U -> (s0_vaddr(2, 0) === 0.U) //d 499 )) 500 501 // accept load flow if dcache ready (tlb is always ready) 502 // TODO: prefetch need writeback to loadQueueFlag 503 s0_out := DontCare 504 s0_out.rsIdx := s0_rsIdx 505 s0_out.vaddr := s0_vaddr 506 s0_out.mask := s0_mask 507 s0_out.uop := s0_uop 508 s0_out.isFirstIssue := s0_isFirstIssue 509 s0_out.hasROBEntry := s0_has_rob_entry 510 s0_out.isPrefetch := s0_prf 511 s0_out.isHWPrefetch := s0_hw_prf 512 s0_out.isFastReplay := s0_fast_rep 513 s0_out.isLoadReplay := s0_ld_rep 514 s0_out.isFastPath := s0_l2l_fwd 515 s0_out.mshrid := s0_mshrid 516 s0_out.uop.cf.exceptionVec(loadAddrMisaligned) := !s0_addr_aligned 517 s0_out.forward_tlDchannel := s0_super_ld_rep_select 518 when(io.tlb.req.valid && s0_isFirstIssue) { 519 s0_out.uop.debugInfo.tlbFirstReqTime := GTimer() 520 }.otherwise{ 521 s0_out.uop.debugInfo.tlbFirstReqTime := s0_uop.debugInfo.tlbFirstReqTime 522 } 523 s0_out.schedIndex := s0_sched_idx 524 525 // load fast replay 526 io.fast_rep_in.ready := (s0_can_go && io.dcache.req.ready && s0_ld_fast_rep_ready) 527 528 // load flow source ready 529 // cache missed load has highest priority 530 // always accept cache missed load flow from load replay queue 531 io.replay.ready := (s0_can_go && io.dcache.req.ready && (s0_ld_rep_ready && !s0_rep_stall || s0_super_ld_rep_select)) 532 533 // accept load flow from rs when: 534 // 1) there is no lsq-replayed load 535 // 2) there is no fast replayed load 536 // 3) there is no high confidence prefetch request 537 io.ldin.ready := (s0_can_go && io.dcache.req.ready && s0_int_iss_ready) 538 539 // for hw prefetch load flow feedback, to be added later 540 // io.prefetch_in.ready := s0_hw_prf_select 541 542 // dcache replacement extra info 543 // TODO: should prefetch load update replacement? 544 io.dcache.replacementUpdated := Mux(s0_ld_rep_select || s0_super_ld_rep_select, io.replay.bits.replacementUpdated, false.B) 545 546 XSDebug(io.dcache.req.fire, 547 p"[DCACHE LOAD REQ] pc ${Hexadecimal(s0_uop.cf.pc)}, vaddr ${Hexadecimal(s0_vaddr)}\n" 548 ) 549 XSDebug(s0_valid, 550 p"S0: pc ${Hexadecimal(s0_out.uop.cf.pc)}, lId ${Hexadecimal(s0_out.uop.lqIdx.asUInt)}, " + 551 p"vaddr ${Hexadecimal(s0_out.vaddr)}, mask ${Hexadecimal(s0_out.mask)}\n") 552 553 // Pipeline 554 // -------------------------------------------------------------------------------- 555 // stage 1 556 // -------------------------------------------------------------------------------- 557 // TLB resp (send paddr to dcache) 558 val s1_valid = RegInit(false.B) 559 val s1_in = Wire(new LqWriteBundle) 560 val s1_out = Wire(new LqWriteBundle) 561 val s1_kill = Wire(Bool()) 562 val s1_can_go = s2_ready 563 val s1_fire = s1_valid && !s1_kill && s1_can_go 564 565 s1_ready := !s1_valid || s1_kill || s2_ready 566 when (s0_fire) { s1_valid := true.B } 567 .elsewhen (s1_fire) { s1_valid := false.B } 568 .elsewhen (s1_kill) { s1_valid := false.B } 569 s1_in := RegEnable(s0_out, s0_fire) 570 571 val s1_fast_rep_dly_err = RegNext(io.fast_rep_in.bits.delayedLoadError) 572 val s1_fast_rep_kill = s1_fast_rep_dly_err && s1_in.isFastReplay 573 val s1_l2l_fwd_dly_err = RegNext(io.l2l_fwd_in.dly_ld_err) 574 val s1_l2l_fwd_kill = s1_l2l_fwd_dly_err && s1_in.isFastPath 575 val s1_late_kill = s1_fast_rep_kill || s1_l2l_fwd_kill 576 val s1_vaddr_hi = Wire(UInt()) 577 val s1_vaddr_lo = Wire(UInt()) 578 val s1_vaddr = Wire(UInt()) 579 val s1_paddr_dup_lsu = Wire(UInt()) 580 val s1_paddr_dup_dcache = Wire(UInt()) 581 val s1_exception = ExceptionNO.selectByFu(s1_out.uop.cf.exceptionVec, lduCfg).asUInt.orR // af & pf exception were modified below. 582 val s1_tlb_miss = io.tlb.resp.bits.miss 583 val s1_prf = s1_in.isPrefetch 584 val s1_hw_prf = s1_in.isHWPrefetch 585 val s1_sw_prf = s1_prf && !s1_hw_prf 586 val s1_tlb_memidx = io.tlb.resp.bits.memidx 587 588 s1_vaddr_hi := s1_in.vaddr(VAddrBits - 1, 6) 589 s1_vaddr_lo := s1_in.vaddr(5, 0) 590 s1_vaddr := Cat(s1_vaddr_hi, s1_vaddr_lo) 591 s1_paddr_dup_lsu := io.tlb.resp.bits.paddr(0) 592 s1_paddr_dup_dcache := io.tlb.resp.bits.paddr(1) 593 594 when (s1_tlb_memidx.is_ld && io.tlb.resp.valid && !s1_tlb_miss && s1_tlb_memidx.idx === s1_in.uop.lqIdx.value) { 595 // printf("load idx = %d\n", s1_tlb_memidx.idx) 596 s1_out.uop.debugInfo.tlbRespTime := GTimer() 597 } 598 599 io.tlb.req_kill := s1_kill 600 io.tlb.resp.ready := true.B 601 602 io.dcache.s1_paddr_dup_lsu <> s1_paddr_dup_lsu 603 io.dcache.s1_paddr_dup_dcache <> s1_paddr_dup_dcache 604 io.dcache.s1_kill := s1_kill || s1_tlb_miss || s1_exception 605 606 // store to load forwarding 607 io.sbuffer.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_prf) 608 io.sbuffer.vaddr := s1_vaddr 609 io.sbuffer.paddr := s1_paddr_dup_lsu 610 io.sbuffer.uop := s1_in.uop 611 io.sbuffer.sqIdx := s1_in.uop.sqIdx 612 io.sbuffer.mask := s1_in.mask 613 io.sbuffer.pc := s1_in.uop.cf.pc // FIXME: remove it 614 615 io.lsq.forward.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_prf) 616 io.lsq.forward.vaddr := s1_vaddr 617 io.lsq.forward.paddr := s1_paddr_dup_lsu 618 io.lsq.forward.uop := s1_in.uop 619 io.lsq.forward.sqIdx := s1_in.uop.sqIdx 620 io.lsq.forward.sqIdxMask := 0.U 621 io.lsq.forward.mask := s1_in.mask 622 io.lsq.forward.pc := s1_in.uop.cf.pc // FIXME: remove it 623 624 // st-ld violation query 625 val s1_nuke = VecInit((0 until StorePipelineWidth).map(w => { 626 io.stld_nuke_query(w).valid && // query valid 627 isAfter(s1_in.uop.robIdx, io.stld_nuke_query(w).bits.robIdx) && // older store 628 // TODO: Fix me when vector instruction 629 (s1_paddr_dup_lsu(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 630 (s1_in.mask & io.stld_nuke_query(w).bits.mask).orR // data mask contain 631 })).asUInt.orR && !s1_tlb_miss 632 633 s1_out := s1_in 634 s1_out.vaddr := s1_vaddr 635 s1_out.paddr := s1_paddr_dup_lsu 636 s1_out.tlbMiss := s1_tlb_miss 637 s1_out.ptwBack := io.tlb.resp.bits.ptwBack 638 s1_out.rsIdx := s1_in.rsIdx 639 s1_out.rep_info.debug := s1_in.uop.debugInfo 640 s1_out.rep_info.nuke := s1_nuke && !s1_sw_prf 641 s1_out.lateKill := s1_late_kill 642 643 when (!s1_late_kill) { 644 // current ori test will cause the case of ldest == 0, below will be modifeid in the future. 645 // af & pf exception were modified 646 s1_out.uop.cf.exceptionVec(loadPageFault) := io.tlb.resp.bits.excp(0).pf.ld 647 s1_out.uop.cf.exceptionVec(loadAccessFault) := io.tlb.resp.bits.excp(0).af.ld 648 } .otherwise { 649 s1_out.uop.cf.exceptionVec(loadAddrMisaligned) := false.B 650 s1_out.uop.cf.exceptionVec(loadAccessFault) := s1_late_kill 651 } 652 653 // pointer chasing 654 val s1_try_ptr_chasing = RegNext(s0_do_try_ptr_chasing, false.B) 655 val s1_ptr_chasing_vaddr = RegEnable(s0_ptr_chasing_vaddr, s0_do_try_ptr_chasing) 656 val s1_fu_op_type_not_ld = WireInit(false.B) 657 val s1_not_fast_match = WireInit(false.B) 658 val s1_addr_mismatch = WireInit(false.B) 659 val s1_addr_misaligned = WireInit(false.B) 660 val s1_ptr_chasing_canceled = WireInit(false.B) 661 val s1_cancel_ptr_chasing = WireInit(false.B) 662 663 s1_kill := s1_late_kill || 664 s1_cancel_ptr_chasing || 665 s1_in.uop.robIdx.needFlush(io.redirect) || 666 RegEnable(s0_kill, false.B, io.ldin.valid || io.replay.valid || io.l2l_fwd_in.valid || io.fast_rep_in.valid) 667 668 if (EnableLoadToLoadForward) { 669 // Sometimes, we need to cancel the load-load forwarding. 670 // These can be put at S0 if timing is bad at S1. 671 // Case 0: CACHE_SET(base + offset) != CACHE_SET(base) (lowest 6-bit addition has an overflow) 672 s1_addr_mismatch := s1_ptr_chasing_vaddr(6) || RegEnable(io.ld_fast_imm(11, 6).orR, s0_do_try_ptr_chasing) 673 // Case 1: the address is misaligned, kill s1 674 s1_addr_misaligned := LookupTree(s1_in.uop.ctrl.fuOpType(1, 0), List( 675 "b00".U -> false.B, //b 676 "b01".U -> (s1_vaddr(0) =/= 0.U), //h 677 "b10".U -> (s1_vaddr(1, 0) =/= 0.U), //w 678 "b11".U -> (s1_vaddr(2, 0) =/= 0.U) //d 679 )) 680 // Case 2: this load-load uop is cancelled 681 s1_ptr_chasing_canceled := !io.ldin.valid 682 683 when (s1_try_ptr_chasing) { 684 s1_cancel_ptr_chasing := s1_addr_mismatch || s1_addr_misaligned || s1_ptr_chasing_canceled 685 686 s1_in.uop := io.ldin.bits.uop 687 s1_in.rsIdx := io.rsIdx 688 s1_in.isFirstIssue := io.isFirstIssue 689 s1_vaddr_lo := s1_ptr_chasing_vaddr(5, 0) 690 s1_paddr_dup_lsu := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 691 s1_paddr_dup_dcache := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 692 693 // recored tlb time when get the data to ensure the correctness of the latency calculation (although it should not record in here, because it does not use tlb) 694 s1_in.uop.debugInfo.tlbFirstReqTime := GTimer() 695 s1_in.uop.debugInfo.tlbRespTime := GTimer() 696 } 697 when (!s1_cancel_ptr_chasing) { 698 s0_ptr_chasing_canceled := s1_try_ptr_chasing && !io.replay.fire && !io.fast_rep_in.fire 699 when (s1_try_ptr_chasing) { 700 io.ldin.ready := true.B 701 } 702 } 703 } 704 705 // pre-calcuate sqIdx mask in s0, then send it to lsq in s1 for forwarding 706 val s1_sqIdx_mask = RegNext(UIntToMask(s0_out.uop.sqIdx.value, StoreQueueSize)) 707 // to enable load-load, sqIdxMask must be calculated based on ldin.uop 708 // If the timing here is not OK, load-load forwarding has to be disabled. 709 // Or we calculate sqIdxMask at RS?? 710 io.lsq.forward.sqIdxMask := s1_sqIdx_mask 711 if (EnableLoadToLoadForward) { 712 when (s1_try_ptr_chasing) { 713 io.lsq.forward.sqIdxMask := UIntToMask(io.ldin.bits.uop.sqIdx.value, StoreQueueSize) 714 } 715 } 716 717 io.forward_mshr.valid := s1_valid && s1_out.forward_tlDchannel 718 io.forward_mshr.mshrid := s1_out.mshrid 719 io.forward_mshr.paddr := s1_out.paddr 720 721 XSDebug(s1_valid, 722 p"S1: pc ${Hexadecimal(s1_out.uop.cf.pc)}, lId ${Hexadecimal(s1_out.uop.lqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 723 p"paddr ${Hexadecimal(s1_out.paddr)}, mmio ${s1_out.mmio}\n") 724 725 // Pipeline 726 // -------------------------------------------------------------------------------- 727 // stage 2 728 // -------------------------------------------------------------------------------- 729 // s2: DCache resp 730 val s2_valid = RegInit(false.B) 731 val s2_in = Wire(new LqWriteBundle) 732 val s2_out = Wire(new LqWriteBundle) 733 val s2_kill = Wire(Bool()) 734 val s2_can_go = s3_ready 735 val s2_fire = s2_valid && !s2_kill && s2_can_go 736 737 s2_kill := s2_in.uop.robIdx.needFlush(io.redirect) 738 s2_ready := !s2_valid || s2_kill || s3_ready 739 when (s1_fire) { s2_valid := true.B } 740 .elsewhen (s2_fire) { s2_valid := false.B } 741 .elsewhen (s2_kill) { s2_valid := false.B } 742 s2_in := RegEnable(s1_out, s1_fire) 743 744 val s2_pmp = WireInit(io.pmp) 745 746 val s2_prf = s2_in.isPrefetch 747 val s2_hw_prf = s2_in.isHWPrefetch 748 749 // exception that may cause load addr to be invalid / illegal 750 // if such exception happen, that inst and its exception info 751 // will be force writebacked to rob 752 val s2_exception_vec = WireInit(s2_in.uop.cf.exceptionVec) 753 when (!s2_in.lateKill) { 754 s2_exception_vec(loadAccessFault) := s2_in.uop.cf.exceptionVec(loadAccessFault) || s2_pmp.ld 755 // soft prefetch will not trigger any exception (but ecc error interrupt may be triggered) 756 when (s2_prf || s2_in.tlbMiss) { 757 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 758 } 759 } 760 val s2_exception = ExceptionNO.selectByFu(s2_exception_vec, lduCfg).asUInt.orR 761 762 val (s2_fwd_frm_d_chan, s2_fwd_data_frm_d_chan) = io.tl_d_channel.forward(s1_valid && s1_out.forward_tlDchannel, s1_out.mshrid, s1_out.paddr) 763 val (s2_fwd_data_valid, s2_fwd_frm_mshr, s2_fwd_data_frm_mshr) = io.forward_mshr.forward() 764 val s2_fwd_frm_d_chan_or_mshr = s2_fwd_data_valid && (s2_fwd_frm_d_chan || s2_fwd_frm_mshr) 765 766 // writeback access fault caused by ecc error / bus error 767 // * ecc data error is slow to generate, so we will not use it until load stage 3 768 // * in load stage 3, an extra signal io.load_error will be used to 769 val s2_actually_mmio = s2_pmp.mmio 770 val s2_mmio = !s2_prf && 771 s2_actually_mmio && 772 !s2_exception && 773 !s2_in.tlbMiss 774 775 val s2_full_fwd = Wire(Bool()) 776 val s2_mem_amb = s2_in.uop.cf.storeSetHit && 777 io.lsq.forward.addrInvalid 778 779 val s2_tlb_miss = s2_in.tlbMiss 780 val s2_fwd_fail = io.lsq.forward.dataInvalid 781 val s2_dcache_miss = io.dcache.resp.bits.miss && 782 !s2_fwd_frm_d_chan_or_mshr && 783 !s2_full_fwd 784 785 val s2_mq_nack = io.dcache.s2_mq_nack && 786 !s2_fwd_frm_d_chan_or_mshr && 787 !s2_full_fwd 788 789 val s2_bank_conflict = io.dcache.s2_bank_conflict && 790 !s2_fwd_frm_d_chan_or_mshr && 791 !s2_full_fwd 792 793 val s2_wpu_pred_fail = io.dcache.s2_wpu_pred_fail && 794 !s2_fwd_frm_d_chan_or_mshr && 795 !s2_full_fwd 796 797 val s2_rar_nack = io.lsq.ldld_nuke_query.req.valid && 798 !io.lsq.ldld_nuke_query.req.ready 799 800 val s2_raw_nack = io.lsq.stld_nuke_query.req.valid && 801 !io.lsq.stld_nuke_query.req.ready 802 // st-ld violation query 803 // NeedFastRecovery Valid when 804 // 1. Fast recovery query request Valid. 805 // 2. Load instruction is younger than requestors(store instructions). 806 // 3. Physical address match. 807 // 4. Data contains. 808 val s2_nuke = VecInit((0 until StorePipelineWidth).map(w => { 809 io.stld_nuke_query(w).valid && // query valid 810 isAfter(s2_in.uop.robIdx, io.stld_nuke_query(w).bits.robIdx) && // older store 811 // TODO: Fix me when vector instruction 812 (s2_in.paddr(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 813 (s2_in.mask & io.stld_nuke_query(w).bits.mask).orR // data mask contain 814 })).asUInt.orR && !s2_tlb_miss || s2_in.rep_info.nuke 815 816 val s2_cache_handled = io.dcache.resp.bits.handled 817 val s2_cache_tag_error = RegNext(io.csrCtrl.cache_error_enable) && 818 io.dcache.resp.bits.tag_error 819 820 val s2_troublem = !s2_exception && 821 !s2_mmio && 822 !s2_prf && 823 !s2_in.lateKill 824 825 io.dcache.resp.ready := true.B 826 val s2_dcache_should_resp = !(s2_in.tlbMiss || s2_exception || s2_mmio || s2_prf || s2_in.lateKill) 827 assert(!(s2_valid && (s2_dcache_should_resp && !io.dcache.resp.valid)), "DCache response got lost") 828 829 // fast replay require 830 val s2_dcache_fast_rep = (s2_mq_nack || !s2_dcache_miss && (s2_bank_conflict || s2_wpu_pred_fail)) 831 val s2_nuke_fast_rep = !s2_mq_nack && 832 !s2_dcache_miss && 833 !s2_bank_conflict && 834 !s2_wpu_pred_fail && 835 !s2_rar_nack && 836 !s2_raw_nack && 837 s2_nuke 838 839 val s2_fast_rep = !s2_mem_amb && 840 !s2_tlb_miss && 841 !s2_fwd_fail && 842 (s2_dcache_fast_rep || s2_nuke_fast_rep) && 843 s2_troublem 844 845 // need allocate new entry 846 val s2_can_query = !s2_mem_amb && 847 !s2_tlb_miss && 848 !s2_fwd_fail && 849 !s2_dcache_fast_rep && 850 s2_troublem 851 852 val s2_data_fwded = s2_dcache_miss && (s2_full_fwd || s2_cache_tag_error) 853 854 // ld-ld violation require 855 io.lsq.ldld_nuke_query.req.valid := s2_valid && s2_can_query 856 io.lsq.ldld_nuke_query.req.bits.uop := s2_in.uop 857 io.lsq.ldld_nuke_query.req.bits.mask := s2_in.mask 858 io.lsq.ldld_nuke_query.req.bits.paddr := s2_in.paddr 859 io.lsq.ldld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid, true.B, !s2_dcache_miss) 860 861 // st-ld violation require 862 io.lsq.stld_nuke_query.req.valid := s2_valid && s2_can_query 863 io.lsq.stld_nuke_query.req.bits.uop := s2_in.uop 864 io.lsq.stld_nuke_query.req.bits.mask := s2_in.mask 865 io.lsq.stld_nuke_query.req.bits.paddr := s2_in.paddr 866 io.lsq.stld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid, true.B, !s2_dcache_miss) 867 868 // merge forward result 869 // lsq has higher priority than sbuffer 870 val s2_fwd_mask = Wire(Vec((VLEN/8), Bool())) 871 val s2_fwd_data = Wire(Vec((VLEN/8), UInt(8.W))) 872 s2_full_fwd := ((~s2_fwd_mask.asUInt).asUInt & s2_in.mask) === 0.U && !io.lsq.forward.dataInvalid 873 // generate XLEN/8 Muxs 874 for (i <- 0 until VLEN / 8) { 875 s2_fwd_mask(i) := io.lsq.forward.forwardMask(i) || io.sbuffer.forwardMask(i) 876 s2_fwd_data(i) := Mux(io.lsq.forward.forwardMask(i), io.lsq.forward.forwardData(i), io.sbuffer.forwardData(i)) 877 } 878 879 XSDebug(s2_fire, "[FWD LOAD RESP] pc %x fwd %x(%b) + %x(%b)\n", 880 s2_in.uop.cf.pc, 881 io.lsq.forward.forwardData.asUInt, io.lsq.forward.forwardMask.asUInt, 882 s2_in.forwardData.asUInt, s2_in.forwardMask.asUInt 883 ) 884 885 // 886 s2_out := s2_in 887 s2_out.data := 0.U // data will be generated in load s3 888 s2_out.uop.ctrl.fpWen := s2_in.uop.ctrl.fpWen && !s2_exception 889 s2_out.mmio := s2_mmio 890 s2_out.uop.ctrl.flushPipe := false.B 891 s2_out.uop.cf.exceptionVec := s2_exception_vec 892 s2_out.forwardMask := s2_fwd_mask 893 s2_out.forwardData := s2_fwd_data 894 s2_out.handledByMSHR := s2_cache_handled 895 s2_out.miss := s2_dcache_miss && s2_troublem 896 s2_out.feedbacked := io.feedback_fast.valid 897 898 // Generate replay signal caused by: 899 // * st-ld violation check 900 // * tlb miss 901 // * dcache replay 902 // * forward data invalid 903 // * dcache miss 904 s2_out.rep_info.mem_amb := s2_mem_amb && s2_troublem 905 s2_out.rep_info.tlb_miss := s2_tlb_miss && s2_troublem 906 s2_out.rep_info.fwd_fail := s2_fwd_fail && s2_troublem 907 s2_out.rep_info.dcache_rep := s2_mq_nack && s2_troublem 908 s2_out.rep_info.dcache_miss := s2_dcache_miss && s2_troublem 909 s2_out.rep_info.bank_conflict := s2_bank_conflict && s2_troublem 910 s2_out.rep_info.wpu_fail := s2_wpu_pred_fail && s2_troublem 911 s2_out.rep_info.rar_nack := s2_rar_nack && s2_troublem 912 s2_out.rep_info.raw_nack := s2_raw_nack && s2_troublem 913 s2_out.rep_info.nuke := s2_nuke && s2_troublem 914 s2_out.rep_info.full_fwd := s2_data_fwded 915 s2_out.rep_info.data_inv_sq_idx := io.lsq.forward.dataInvalidSqIdx 916 s2_out.rep_info.addr_inv_sq_idx := io.lsq.forward.addrInvalidSqIdx 917 s2_out.rep_info.rep_carry := io.dcache.resp.bits.replayCarry 918 s2_out.rep_info.mshr_id := io.dcache.resp.bits.mshr_id 919 s2_out.rep_info.last_beat := s2_in.paddr(log2Up(refillBytes)) 920 s2_out.rep_info.debug := s2_in.uop.debugInfo 921 922 // if forward fail, replay this inst from fetch 923 val debug_fwd_fail_rep = s2_fwd_fail && !s2_troublem && !s2_in.tlbMiss 924 // if ld-ld violation is detected, replay from this inst from fetch 925 val debug_ldld_nuke_rep = false.B // s2_ldld_violation && !s2_mmio && !s2_is_prefetch && !s2_in.tlbMiss 926 // io.out.bits.uop.ctrl.replayInst := false.B 927 928 // to be removed 929 io.feedback_fast.valid := s2_valid && // inst is valid 930 !s2_in.isLoadReplay && // already feedbacked 931 io.lq_rep_full && // LoadQueueReplay is full 932 s2_out.rep_info.need_rep && // need replay 933 !s2_exception && // no exception is triggered 934 !s2_hw_prf // not hardware prefetch 935 io.feedback_fast.bits.hit := false.B 936 io.feedback_fast.bits.flushState := s2_in.ptwBack 937 io.feedback_fast.bits.rsIdx := s2_in.rsIdx 938 io.feedback_fast.bits.sourceType := RSFeedbackType.lrqFull 939 io.feedback_fast.bits.dataInvalidSqIdx := DontCare 940 941 // fast wakeup 942 io.fast_uop.valid := RegNext( 943 !io.dcache.s1_disable_fast_wakeup && 944 s1_valid && 945 !s1_kill && 946 !io.tlb.resp.bits.miss && 947 !io.lsq.forward.dataInvalidFast 948 ) && (s2_valid && !s2_out.rep_info.need_rep && !s2_mmio) 949 io.fast_uop.bits := RegNext(s1_out.uop) 950 951 // 952 io.s2_ptr_chasing := RegEnable(s1_try_ptr_chasing && !s1_cancel_ptr_chasing, s1_fire) 953 954 io.prefetch_train.valid := s2_valid && !s2_actually_mmio && !s2_in.tlbMiss 955 io.prefetch_train.bits.fromLsPipelineBundle(s2_in) 956 io.prefetch_train.bits.miss := io.dcache.resp.bits.miss // TODO: use trace with bank conflict? 957 io.prefetch_train.bits.meta_prefetch := io.dcache.resp.bits.meta_prefetch 958 io.prefetch_train.bits.meta_access := io.dcache.resp.bits.meta_access 959 960 961 io.prefetch_train_l1.valid := s2_valid && !s2_actually_mmio 962 io.prefetch_train_l1.bits.fromLsPipelineBundle(s2_in) 963 io.prefetch_train_l1.bits.miss := io.dcache.resp.bits.miss 964 io.prefetch_train_l1.bits.meta_prefetch := io.dcache.resp.bits.meta_prefetch 965 io.prefetch_train_l1.bits.meta_access := io.dcache.resp.bits.meta_access 966 if (env.FPGAPlatform){ 967 io.dcache.s0_pc := DontCare 968 io.dcache.s1_pc := DontCare 969 io.dcache.s2_pc := DontCare 970 }else{ 971 io.dcache.s0_pc := s0_out.uop.cf.pc 972 io.dcache.s1_pc := s1_out.uop.cf.pc 973 io.dcache.s2_pc := s2_out.uop.cf.pc 974 } 975 io.dcache.s2_kill := s2_pmp.ld || s2_actually_mmio || s2_kill 976 977 val s1_ld_left_fire = s1_valid && !s1_kill && s2_ready 978 val s2_ld_valid_dup = RegInit(0.U(6.W)) 979 s2_ld_valid_dup := 0x0.U(6.W) 980 when (s1_ld_left_fire && !s1_out.isHWPrefetch) { s2_ld_valid_dup := 0x3f.U(6.W) } 981 when (s1_kill || s1_out.isHWPrefetch) { s2_ld_valid_dup := 0x0.U(6.W) } 982 assert(RegNext((s2_valid === s2_ld_valid_dup(0)) || RegNext(s1_out.isHWPrefetch))) 983 984 // Pipeline 985 // -------------------------------------------------------------------------------- 986 // stage 3 987 // -------------------------------------------------------------------------------- 988 // writeback and update load queue 989 val s3_valid = RegNext(s2_valid && !s2_out.isHWPrefetch && !s2_out.uop.robIdx.needFlush(io.redirect)) 990 val s3_in = RegEnable(s2_out, s2_fire) 991 val s3_out = Wire(Valid(new ExuOutput)) 992 val s3_dcache_rep = RegEnable(s2_dcache_fast_rep && s2_troublem, s2_fire) 993 val s3_ld_valid_dup = RegEnable(s2_ld_valid_dup, s2_fire) 994 val s3_fast_rep = Wire(Bool()) 995 val s3_troublem = RegNext(s2_troublem) 996 val s3_kill = s3_in.uop.robIdx.needFlush(io.redirect) 997 s3_ready := !s3_valid || s3_kill || io.ldout.ready 998 999 // forwrad last beat 1000 val (s3_fwd_frm_d_chan, s3_fwd_data_frm_d_chan) = io.tl_d_channel.forward(s2_valid && s2_out.forward_tlDchannel, s2_out.mshrid, s2_out.paddr) 1001 val s3_fwd_data_valid = RegEnable(s2_fwd_data_valid, s2_valid) 1002 val s3_fwd_frm_d_chan_valid = (s3_fwd_frm_d_chan && s3_fwd_data_valid) 1003 val s3_nuke = VecInit((0 until StorePipelineWidth).map(w => { 1004 io.stld_nuke_query(w).valid && // query valid 1005 isAfter(s3_in.uop.robIdx, io.stld_nuke_query(w).bits.robIdx) && // older store 1006 // TODO: Fix me when vector instruction 1007 (s3_in.paddr(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 1008 (s3_in.mask & io.stld_nuke_query(w).bits.mask).orR // data mask contain 1009 })).asUInt.orR && !s3_in.tlbMiss || s3_in.rep_info.nuke 1010 1011 1012 // s3 load fast replay 1013 io.fast_rep_out.valid := s3_valid && s3_fast_rep && !s3_in.uop.robIdx.needFlush(io.redirect) 1014 io.fast_rep_out.bits := s3_in 1015 1016 io.lsq.ldin.valid := s3_valid && (!s3_fast_rep || !io.fast_rep_out.ready) && !s3_in.feedbacked && !s3_in.lateKill 1017 io.lsq.ldin.bits := s3_in 1018 io.lsq.ldin.bits.miss := s3_in.miss && !s3_fwd_frm_d_chan_valid 1019 1020 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1021 io.lsq.ldin.bits.data_wen_dup := s3_ld_valid_dup.asBools 1022 io.lsq.ldin.bits.replacementUpdated := io.dcache.resp.bits.replacementUpdated 1023 io.lsq.ldin.bits.missDbUpdated := RegNext(s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated) 1024 1025 val s3_dly_ld_err = 1026 if (EnableAccurateLoadError) { 1027 (s3_in.lateKill || io.dcache.resp.bits.error_delayed) && RegNext(io.csrCtrl.cache_error_enable) 1028 } else { 1029 WireInit(false.B) 1030 } 1031 io.s3_dly_ld_err := false.B // s3_dly_ld_err && s3_valid 1032 io.fast_rep_out.bits.delayedLoadError := s3_dly_ld_err 1033 io.lsq.ldin.bits.dcacheRequireReplay := s3_dcache_rep 1034 1035 val s3_vp_match_fail = RegNext(io.lsq.forward.matchInvalid || io.sbuffer.matchInvalid) && s3_troublem 1036 val s3_ldld_rep_inst = 1037 io.lsq.ldld_nuke_query.resp.valid && 1038 io.lsq.ldld_nuke_query.resp.bits.rep_frm_fetch && 1039 RegNext(io.csrCtrl.ldld_vio_check_enable) 1040 1041 val s3_rep_info = WireInit(s3_in.rep_info) 1042 s3_rep_info.wpu_fail := s3_in.rep_info.wpu_fail && !s3_fwd_frm_d_chan_valid && s3_troublem 1043 s3_rep_info.bank_conflict := s3_in.rep_info.bank_conflict && !s3_fwd_frm_d_chan_valid && s3_troublem 1044 s3_rep_info.dcache_miss := s3_in.rep_info.dcache_miss && !s3_fwd_frm_d_chan_valid && s3_troublem 1045 s3_rep_info.nuke := s3_nuke && s3_troublem 1046 val s3_rep_frm_fetch = s3_vp_match_fail || s3_ldld_rep_inst 1047 val s3_sel_rep_cause = PriorityEncoderOH(s3_rep_info.cause.asUInt) 1048 val s3_force_rep = s3_sel_rep_cause(LoadReplayCauses.C_TM) && 1049 !s3_in.uop.cf.exceptionVec(loadAddrMisaligned) && 1050 s3_troublem 1051 1052 val s3_exception = ExceptionNO.selectByFu(s3_in.uop.cf.exceptionVec, lduCfg).asUInt.orR 1053 when ((s3_exception || s3_dly_ld_err || s3_rep_frm_fetch) && !s3_force_rep) { 1054 io.lsq.ldin.bits.rep_info.cause := 0.U.asTypeOf(s3_rep_info.cause.cloneType) 1055 } .otherwise { 1056 io.lsq.ldin.bits.rep_info.cause := VecInit(s3_sel_rep_cause.asBools) 1057 } 1058 1059 // Int load, if hit, will be writebacked at s3 1060 s3_out.valid := s3_valid && !io.lsq.ldin.bits.rep_info.need_rep && !s3_in.mmio 1061 s3_out.bits.uop := s3_in.uop 1062 s3_out.bits.uop.cf.exceptionVec(loadAccessFault) := s3_dly_ld_err || s3_in.uop.cf.exceptionVec(loadAccessFault) 1063 s3_out.bits.uop.ctrl.replayInst := s3_rep_frm_fetch 1064 s3_out.bits.data := s3_in.data 1065 s3_out.bits.redirectValid := false.B 1066 s3_out.bits.redirect := DontCare 1067 s3_out.bits.debug.isMMIO := s3_in.mmio 1068 s3_out.bits.debug.isPerfCnt := false.B 1069 s3_out.bits.debug.paddr := s3_in.paddr 1070 s3_out.bits.debug.vaddr := s3_in.vaddr 1071 s3_out.bits.fflags := DontCare 1072 1073 when (s3_force_rep) { 1074 s3_out.bits.uop.cf.exceptionVec := 0.U.asTypeOf(s3_in.uop.cf.exceptionVec.cloneType) 1075 } 1076 1077 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1078 1079 io.lsq.ldin.bits.uop := s3_out.bits.uop 1080 1081 val s3_revoke = s3_exception || io.lsq.ldin.bits.rep_info.need_rep 1082 io.lsq.ldld_nuke_query.revoke := s3_revoke 1083 io.lsq.stld_nuke_query.revoke := s3_revoke 1084 1085 // feedback slow 1086 s3_fast_rep := RegNext(s2_fast_rep) && 1087 !s3_in.feedbacked && 1088 !s3_in.lateKill && 1089 !s3_rep_frm_fetch && 1090 !s3_exception 1091 1092 val s3_fb_no_waiting = !s3_in.isLoadReplay && !(s3_fast_rep && io.fast_rep_out.ready) && !s3_in.feedbacked 1093 1094 // 1095 io.feedback_slow.valid := s3_valid && !s3_in.uop.robIdx.needFlush(io.redirect) && s3_fb_no_waiting 1096 io.feedback_slow.bits.hit := !io.lsq.ldin.bits.rep_info.need_rep || io.lsq.ldin.ready 1097 io.feedback_slow.bits.flushState := s3_in.ptwBack 1098 io.feedback_slow.bits.rsIdx := s3_in.rsIdx 1099 io.feedback_slow.bits.sourceType := RSFeedbackType.lrqFull 1100 io.feedback_slow.bits.dataInvalidSqIdx := DontCare 1101 1102 val s3_ld_wb_meta = Mux(s3_out.valid, s3_out.bits, io.lsq.uncache.bits) 1103 1104 // data from load queue refill 1105 val s3_ld_raw_data_frm_uncache = io.lsq.ld_raw_data 1106 val s3_merged_data_frm_uncache = s3_ld_raw_data_frm_uncache.mergedData() 1107 val s3_picked_data_frm_uncache = LookupTree(s3_ld_raw_data_frm_uncache.addrOffset, List( 1108 "b000".U -> s3_merged_data_frm_uncache(63, 0), 1109 "b001".U -> s3_merged_data_frm_uncache(63, 8), 1110 "b010".U -> s3_merged_data_frm_uncache(63, 16), 1111 "b011".U -> s3_merged_data_frm_uncache(63, 24), 1112 "b100".U -> s3_merged_data_frm_uncache(63, 32), 1113 "b101".U -> s3_merged_data_frm_uncache(63, 40), 1114 "b110".U -> s3_merged_data_frm_uncache(63, 48), 1115 "b111".U -> s3_merged_data_frm_uncache(63, 56) 1116 )) 1117 val s3_ld_data_frm_uncache = rdataHelper(s3_ld_raw_data_frm_uncache.uop, s3_picked_data_frm_uncache) 1118 1119 // data from dcache hit 1120 val s3_ld_raw_data_frm_cache = Wire(new LoadDataFromDcacheBundle) 1121 s3_ld_raw_data_frm_cache.respDcacheData := io.dcache.resp.bits.data_delayed 1122 s3_ld_raw_data_frm_cache.forwardMask := RegEnable(s2_fwd_mask, s2_valid) 1123 s3_ld_raw_data_frm_cache.forwardData := RegEnable(s2_fwd_data, s2_valid) 1124 s3_ld_raw_data_frm_cache.uop := RegEnable(s2_out.uop, s2_valid) 1125 s3_ld_raw_data_frm_cache.addrOffset := RegEnable(s2_out.paddr(3, 0), s2_valid) 1126 s3_ld_raw_data_frm_cache.forward_D := RegEnable(s2_fwd_frm_d_chan, s2_valid) || s3_fwd_frm_d_chan_valid 1127 s3_ld_raw_data_frm_cache.forwardData_D := Mux(s3_fwd_frm_d_chan_valid, s3_fwd_data_frm_d_chan, RegEnable(s2_fwd_data_frm_d_chan, s2_valid)) 1128 s3_ld_raw_data_frm_cache.forward_mshr := RegEnable(s2_fwd_frm_mshr, s2_valid) 1129 s3_ld_raw_data_frm_cache.forwardData_mshr := RegEnable(s2_fwd_data_frm_mshr, s2_valid) 1130 s3_ld_raw_data_frm_cache.forward_result_valid := RegEnable(s2_fwd_data_valid, s2_valid) 1131 1132 val s3_merged_data_frm_cache = s3_ld_raw_data_frm_cache.mergedData() 1133 val s3_picked_data_frm_cache = LookupTree(s3_ld_raw_data_frm_cache.addrOffset, List( 1134 "b0000".U -> s3_merged_data_frm_cache(63, 0), 1135 "b0001".U -> s3_merged_data_frm_cache(63, 8), 1136 "b0010".U -> s3_merged_data_frm_cache(63, 16), 1137 "b0011".U -> s3_merged_data_frm_cache(63, 24), 1138 "b0100".U -> s3_merged_data_frm_cache(63, 32), 1139 "b0101".U -> s3_merged_data_frm_cache(63, 40), 1140 "b0110".U -> s3_merged_data_frm_cache(63, 48), 1141 "b0111".U -> s3_merged_data_frm_cache(63, 56), 1142 "b1000".U -> s3_merged_data_frm_cache(127, 64), 1143 "b1001".U -> s3_merged_data_frm_cache(127, 72), 1144 "b1010".U -> s3_merged_data_frm_cache(127, 80), 1145 "b1011".U -> s3_merged_data_frm_cache(127, 88), 1146 "b1100".U -> s3_merged_data_frm_cache(127, 96), 1147 "b1101".U -> s3_merged_data_frm_cache(127, 104), 1148 "b1110".U -> s3_merged_data_frm_cache(127, 112), 1149 "b1111".U -> s3_merged_data_frm_cache(127, 120) 1150 )) 1151 val s3_ld_data_frm_cache = rdataHelper(s3_ld_raw_data_frm_cache.uop, s3_picked_data_frm_cache) 1152 1153 // FIXME: add 1 cycle delay ? 1154 io.lsq.uncache.ready := !s3_out.valid 1155 io.ldout.bits := s3_ld_wb_meta 1156 io.ldout.bits.data := Mux(s3_out.valid, s3_ld_data_frm_cache, s3_ld_data_frm_uncache) 1157 io.ldout.valid := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) || 1158 io.lsq.uncache.valid && !io.lsq.uncache.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid 1159 1160 1161 // fast load to load forward 1162 io.l2l_fwd_out.valid := s3_out.valid && !s3_in.lateKill 1163 io.l2l_fwd_out.data := s3_ld_data_frm_cache 1164 io.l2l_fwd_out.dly_ld_err := s3_dly_ld_err // ecc delayed error 1165 1166 // trigger 1167 val last_valid_data = RegNext(RegEnable(io.ldout.bits.data, io.ldout.fire)) 1168 val hit_ld_addr_trig_hit_vec = Wire(Vec(3, Bool())) 1169 val lq_ld_addr_trig_hit_vec = io.lsq.trigger.lqLoadAddrTriggerHitVec 1170 (0 until 3).map{i => { 1171 val tdata2 = RegNext(io.trigger(i).tdata2) 1172 val matchType = RegNext(io.trigger(i).matchType) 1173 val tEnable = RegNext(io.trigger(i).tEnable) 1174 1175 hit_ld_addr_trig_hit_vec(i) := TriggerCmp(RegNext(s2_out.vaddr), tdata2, matchType, tEnable) 1176 io.trigger(i).addrHit := Mux(s3_out.valid, hit_ld_addr_trig_hit_vec(i), lq_ld_addr_trig_hit_vec(i)) 1177 io.trigger(i).lastDataHit := TriggerCmp(last_valid_data, tdata2, matchType, tEnable) 1178 }} 1179 io.lsq.trigger.hitLoadAddrTriggerHitVec := hit_ld_addr_trig_hit_vec 1180 1181 // FIXME: please move this part to LoadQueueReplay 1182 io.debug_ls := DontCare 1183 1184 // Topdown 1185 io.lsTopdownInfo.s1.robIdx := s1_in.uop.robIdx.value 1186 io.lsTopdownInfo.s1.vaddr_valid := s1_valid && s1_in.hasROBEntry 1187 io.lsTopdownInfo.s1.vaddr_bits := s1_vaddr 1188 io.lsTopdownInfo.s2.robIdx := s2_in.uop.robIdx.value 1189 io.lsTopdownInfo.s2.paddr_valid := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss 1190 io.lsTopdownInfo.s2.paddr_bits := s2_in.paddr 1191 io.lsTopdownInfo.s2.first_real_miss := io.dcache.resp.bits.real_miss 1192 io.lsTopdownInfo.s2.cache_miss_en := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated 1193 1194 // perf cnt 1195 XSPerfAccumulate("s0_in_valid", io.ldin.valid) 1196 XSPerfAccumulate("s0_in_block", io.ldin.valid && !io.ldin.fire) 1197 XSPerfAccumulate("s0_in_fire_first_issue", s0_valid && s0_isFirstIssue) 1198 XSPerfAccumulate("s0_lsq_fire_first_issue", io.replay.fire) 1199 XSPerfAccumulate("s0_ldu_fire_first_issue", io.ldin.fire && s0_isFirstIssue) 1200 XSPerfAccumulate("s0_fast_replay_issue", io.fast_rep_in.fire) 1201 XSPerfAccumulate("s0_stall_out", s0_valid && !s0_can_go) 1202 XSPerfAccumulate("s0_stall_dcache", s0_valid && !io.dcache.req.ready) 1203 XSPerfAccumulate("s0_addr_spec_success", s0_fire && s0_vaddr(VAddrBits-1, 12) === io.ldin.bits.src(0)(VAddrBits-1, 12)) 1204 XSPerfAccumulate("s0_addr_spec_failed", s0_fire && s0_vaddr(VAddrBits-1, 12) =/= io.ldin.bits.src(0)(VAddrBits-1, 12)) 1205 XSPerfAccumulate("s0_addr_spec_success_once", s0_fire && s0_vaddr(VAddrBits-1, 12) === io.ldin.bits.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 1206 XSPerfAccumulate("s0_addr_spec_failed_once", s0_fire && s0_vaddr(VAddrBits-1, 12) =/= io.ldin.bits.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 1207 XSPerfAccumulate("s0_forward_tl_d_channel", s0_out.forward_tlDchannel) 1208 XSPerfAccumulate("s0_hardware_prefetch_fire", s0_fire && s0_hw_prf_select) 1209 XSPerfAccumulate("s0_software_prefetch_fire", s0_fire && s0_prf && s0_int_iss_select) 1210 XSPerfAccumulate("s0_hardware_prefetch_blocked", io.prefetch_req.valid && !s0_hw_prf_select) 1211 XSPerfAccumulate("s0_hardware_prefetch_total", io.prefetch_req.valid) 1212 1213 XSPerfAccumulate("s1_in_valid", s1_valid) 1214 XSPerfAccumulate("s1_in_fire", s1_fire) 1215 XSPerfAccumulate("s1_in_fire_first_issue", s1_fire && s1_in.isFirstIssue) 1216 XSPerfAccumulate("s1_tlb_miss", s1_fire && s1_tlb_miss) 1217 XSPerfAccumulate("s1_tlb_miss_first_issue", s1_fire && s1_tlb_miss && s1_in.isFirstIssue) 1218 XSPerfAccumulate("s1_stall_out", s1_valid && !s1_can_go) 1219 XSPerfAccumulate("s1_late_kill", s1_valid && s1_fast_rep_kill) 1220 1221 XSPerfAccumulate("s2_in_valid", s2_valid) 1222 XSPerfAccumulate("s2_in_fire", s2_fire) 1223 XSPerfAccumulate("s2_in_fire_first_issue", s2_fire && s2_in.isFirstIssue) 1224 XSPerfAccumulate("s2_dcache_miss", s2_fire && io.dcache.resp.bits.miss) 1225 XSPerfAccumulate("s2_dcache_miss_first_issue", s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1226 XSPerfAccumulate("s2_dcache_real_miss_first_issue", s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1227 XSPerfAccumulate("s2_full_forward", s2_fire && s2_full_fwd) 1228 XSPerfAccumulate("s2_dcache_miss_full_forward", s2_fire && s2_dcache_miss) 1229 XSPerfAccumulate("s2_fwd_frm_d_can", s2_valid && s2_fwd_frm_d_chan) 1230 XSPerfAccumulate("s2_fwd_frm_d_chan_or_mshr", s2_valid && s2_fwd_frm_d_chan_or_mshr) 1231 XSPerfAccumulate("s2_stall_out", s2_fire && !s2_can_go) 1232 XSPerfAccumulate("s2_prefetch", s2_fire && s2_prf) 1233 XSPerfAccumulate("s2_prefetch_ignored", s2_fire && s2_prf && s2_mq_nack) // ignore prefetch for mshr full / miss req port conflict 1234 XSPerfAccumulate("s2_prefetch_miss", s2_fire && s2_prf && io.dcache.resp.bits.miss) // prefetch req miss in l1 1235 XSPerfAccumulate("s2_prefetch_hit", s2_fire && s2_prf && !io.dcache.resp.bits.miss) // prefetch req hit in l1 1236 XSPerfAccumulate("s2_prefetch_accept", s2_fire && s2_prf && io.dcache.resp.bits.miss && !s2_mq_nack) // prefetch a missed line in l1, and l1 accepted it 1237 XSPerfAccumulate("s2_forward_req", s2_fire && s2_in.forward_tlDchannel) 1238 XSPerfAccumulate("s2_successfully_forward_channel_D", s2_fire && s2_fwd_frm_d_chan && s2_fwd_data_valid) 1239 XSPerfAccumulate("s2_successfully_forward_mshr", s2_fire && s2_fwd_frm_mshr && s2_fwd_data_valid) 1240 1241 XSPerfAccumulate("s3_fwd_frm_d_chan", s3_valid && s3_fwd_frm_d_chan_valid) 1242 1243 XSPerfAccumulate("load_to_load_forward", s1_try_ptr_chasing && !s1_ptr_chasing_canceled) 1244 XSPerfAccumulate("load_to_load_forward_try", s1_try_ptr_chasing) 1245 XSPerfAccumulate("load_to_load_forward_fail", s1_cancel_ptr_chasing) 1246 XSPerfAccumulate("load_to_load_forward_fail_cancelled", s1_cancel_ptr_chasing && s1_ptr_chasing_canceled) 1247 XSPerfAccumulate("load_to_load_forward_fail_wakeup_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && s1_not_fast_match) 1248 XSPerfAccumulate("load_to_load_forward_fail_op_not_ld", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && s1_fu_op_type_not_ld) 1249 XSPerfAccumulate("load_to_load_forward_fail_addr_align", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && s1_addr_misaligned) 1250 XSPerfAccumulate("load_to_load_forward_fail_set_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && !s1_addr_misaligned && s1_addr_mismatch) 1251 1252 // bug lyq: some signals in perfEvents are no longer suitable for the current MemBlock design 1253 // hardware performance counter 1254 val perfEvents = Seq( 1255 ("load_s0_in_fire ", s0_fire ), 1256 ("load_to_load_forward ", s1_fire && s1_try_ptr_chasing && !s1_ptr_chasing_canceled ), 1257 ("stall_dcache ", s0_valid && s0_can_go && !io.dcache.req.ready ), 1258 ("load_s1_in_fire ", s0_fire ), 1259 ("load_s1_tlb_miss ", s1_fire && io.tlb.resp.bits.miss ), 1260 ("load_s2_in_fire ", s1_fire ), 1261 ("load_s2_dcache_miss ", s2_fire && io.dcache.resp.bits.miss ), 1262 ) 1263 generatePerfEvent() 1264 1265 when(io.ldout.fire){ 1266 XSDebug("ldout %x\n", io.ldout.bits.uop.cf.pc) 1267 } 1268 // end 1269}