1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.Bundles.{DynInst, MemExuInput, MemExuOutput} 27import xiangshan.backend.fu.PMPRespBundle 28import xiangshan.backend.fu.FuConfig._ 29import xiangshan.backend.ctrlblock.{DebugLsInfoBundle, LsTopdownInfo} 30import xiangshan.backend.rob.RobPtr 31import xiangshan.backend.fu._ 32import xiangshan.cache._ 33import xiangshan.cache.wpu.ReplayCarry 34import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp} 35import xiangshan.mem.mdp._ 36 37class HybridUnit(implicit p: Parameters) extends XSModule 38 with HasLoadHelper 39 with HasPerfEvents 40 with HasDCacheParameters 41 with HasCircularQueuePtrHelper 42{ 43 val io = IO(new Bundle() { 44 // control 45 val redirect = Flipped(ValidIO(new Redirect)) 46 val csrCtrl = Flipped(new CustomCSRCtrlIO) 47 48 // flow in 49 val lsin = Flipped(Decoupled(new MemExuInput)) 50 val rsIdx = Input(UInt()) 51 val isFirstIssue = Input(Bool()) 52 53 // flow out 54 val out_to_iq = DecoupledIO(new MemExuOutput) 55 val out_to_rob = DecoupledIO(new MemExuOutput) 56 57 val ldu_io = new Bundle() { 58 // data path 59 val sbuffer = new LoadForwardQueryIO 60 val lsq = new LoadToLsqIO 61 val tl_d_channel = Input(new DcacheToLduForwardIO) 62 val forward_mshr = Flipped(new LduToMissqueueForwardIO) 63 val refill = Flipped(ValidIO(new Refill)) 64 val l2_hint = Input(Valid(new L2ToL1Hint)) 65 66 // fast wakeup 67 val fast_uop = ValidIO(new DynInst) // early wakeup signal generated in load_s1, send to RS in load_s2 68 69 // trigger 70 val trigger = Vec(3, new LoadUnitTriggerIO) 71 72 // prefetch 73 val prefetch_train = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to sms 74 val prefetch_train_l1 = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to stream & stride 75 val prefetch_req = Flipped(ValidIO(new L1PrefetchReq)) // hardware prefetch to l1 cache req 76 val canAcceptLowConfPrefetch = Output(Bool()) 77 val canAcceptHighConfPrefetch = Output(Bool()) 78 79 // load to load fast path 80 val l2l_fwd_in = Input(new LoadToLoadIO) 81 val l2l_fwd_out = Output(new LoadToLoadIO) 82 83 val ld_fast_match = Input(Bool()) 84 val ld_fast_fuOpType = Input(UInt()) 85 val ld_fast_imm = Input(UInt(12.W)) 86 87 // iq cancel 88 val ldCancel = Output(new LoadCancelIO()) // use to cancel the uops waked by this load, and cancel load 89 90 // load ecc error 91 val s3_dly_ld_err = Output(Bool()) // Note that io.s3_dly_ld_err and io.lsq.s3_dly_ld_err is different 92 93 // schedule error query 94 val stld_nuke_query = Flipped(Vec(StorePipelineWidth, Valid(new StoreNukeQueryIO))) 95 96 // queue-based replay 97 val replay = Flipped(Decoupled(new LsPipelineBundle)) 98 val lq_rep_full = Input(Bool()) 99 100 // misc 101 val s2_ptr_chasing = Output(Bool()) // provide right pc for hw prefetch 102 103 // Load fast replay path 104 val fast_rep_in = Flipped(Decoupled(new LqWriteBundle)) 105 val fast_rep_out = Decoupled(new LqWriteBundle) 106 107 // perf 108 val debug_ls = Output(new DebugLsInfoBundle) 109 val lsTopdownInfo = Output(new LsTopdownInfo) 110 val correctMissTrain = Input(Bool()) 111 } 112 113 val stu_io = new Bundle() { 114 val issue = Valid(new MemExuInput) 115 val lsq = ValidIO(new LsPipelineBundle) 116 val lsq_replenish = Output(new LsPipelineBundle()) 117 val stld_nuke_query = Valid(new StoreNukeQueryIO) 118 val st_mask_out = Valid(new StoreMaskBundle) 119 val debug_ls = Output(new DebugLsInfoBundle) 120 } 121 122 // data path 123 val tlb = new TlbRequestIO(2) 124 val pmp = Flipped(new PMPRespBundle()) // arrive same to tlb now 125 val dcache = new DCacheLoadIO 126 127 // rs feedback 128 val feedback_fast = ValidIO(new RSFeedback) // stage 2 129 val feedback_slow = ValidIO(new RSFeedback) // stage 3 130 }) 131 132 val s1_ready, s2_ready, s3_ready, sx_can_go = WireInit(false.B) 133 134 // Pipeline 135 // -------------------------------------------------------------------------------- 136 // stage 0 137 // -------------------------------------------------------------------------------- 138 // generate addr, use addr to query DCache and DTLB 139 val s0_valid = Wire(Bool()) 140 val s0_kill = Wire(Bool()) 141 val s0_vaddr = Wire(UInt(VAddrBits.W)) 142 val s0_mask = Wire(UInt((VLEN/8).W)) 143 val s0_uop = Wire(new DynInst) 144 val s0_has_rob_entry = Wire(Bool()) 145 val s0_rsIdx = Wire(UInt(log2Up(MemIQSizeMax).W)) 146 val s0_mshrid = Wire(UInt()) 147 val s0_try_l2l = Wire(Bool()) 148 val s0_rep_carry = Wire(new ReplayCarry(nWays)) 149 val s0_isFirstIssue = Wire(Bool()) 150 val s0_fast_rep = Wire(Bool()) 151 val s0_ld_rep = Wire(Bool()) 152 val s0_l2l_fwd = Wire(Bool()) 153 val s0_sched_idx = Wire(UInt()) 154 val s0_can_go = s1_ready 155 val s0_fire = s0_valid && s0_can_go 156 val s0_out = Wire(new LqWriteBundle) 157 158 // load flow select/gen 159 // src0: super load replayed by LSQ (cache miss replay) (io.ldu_io.replay) 160 // src1: fast load replay (io.ldu_io.fast_rep_in) 161 // src2: load replayed by LSQ (io.ldu_io.replay) 162 // src3: hardware prefetch from prefetchor (high confidence) (io.prefetch) 163 // src4: int read / software prefetch first issue from RS (io.in) 164 // src5: vec read first issue from RS (TODO) 165 // src6: load try pointchaising when no issued or replayed load (io.fastpath) 166 // src7: hardware prefetch from prefetchor (high confidence) (io.prefetch) 167 // priority: high to low 168 val s0_ld_flow = FuType.isLoad(s0_uop.fuType) 169 val s0_rep_stall = io.lsin.valid && isAfter(io.ldu_io.replay.bits.uop.robIdx, io.lsin.bits.uop.robIdx) 170 val s0_super_ld_rep_valid = io.ldu_io.replay.valid && io.ldu_io.replay.bits.forward_tlDchannel 171 val s0_ld_fast_rep_valid = io.ldu_io.fast_rep_in.valid 172 val s0_ld_rep_valid = io.ldu_io.replay.valid && !io.ldu_io.replay.bits.forward_tlDchannel && !s0_rep_stall 173 val s0_high_conf_prf_valid = io.ldu_io.prefetch_req.valid && io.ldu_io.prefetch_req.bits.confidence > 0.U 174 val s0_int_iss_valid = io.lsin.valid // int flow first issue or software prefetch 175 val s0_vec_iss_valid = WireInit(false.B) // TODO 176 val s0_l2l_fwd_valid = io.ldu_io.l2l_fwd_in.valid && io.ldu_io.ld_fast_match 177 val s0_low_conf_prf_valid = io.ldu_io.prefetch_req.valid && io.ldu_io.prefetch_req.bits.confidence === 0.U 178 dontTouch(s0_super_ld_rep_valid) 179 dontTouch(s0_ld_fast_rep_valid) 180 dontTouch(s0_ld_rep_valid) 181 dontTouch(s0_high_conf_prf_valid) 182 dontTouch(s0_int_iss_valid) 183 dontTouch(s0_vec_iss_valid) 184 dontTouch(s0_l2l_fwd_valid) 185 dontTouch(s0_low_conf_prf_valid) 186 187 // load flow source ready 188 val s0_super_ld_rep_ready = WireInit(true.B) 189 val s0_ld_fast_rep_ready = !s0_super_ld_rep_valid 190 val s0_ld_rep_ready = !s0_super_ld_rep_valid && 191 !s0_ld_fast_rep_valid 192 val s0_high_conf_prf_ready = !s0_super_ld_rep_valid && 193 !s0_ld_fast_rep_valid && 194 !s0_ld_rep_valid 195 196 val s0_int_iss_ready = !s0_super_ld_rep_valid && 197 !s0_ld_fast_rep_valid && 198 !s0_ld_rep_valid && 199 !s0_high_conf_prf_valid 200 201 val s0_vec_iss_ready = !s0_super_ld_rep_valid && 202 !s0_ld_fast_rep_valid && 203 !s0_ld_rep_valid && 204 !s0_high_conf_prf_valid && 205 !s0_int_iss_valid 206 207 val s0_l2l_fwd_ready = !s0_super_ld_rep_valid && 208 !s0_ld_fast_rep_valid && 209 !s0_ld_rep_valid && 210 !s0_high_conf_prf_valid && 211 !s0_int_iss_valid && 212 !s0_vec_iss_valid 213 214 val s0_low_conf_prf_ready = !s0_super_ld_rep_valid && 215 !s0_ld_fast_rep_valid && 216 !s0_ld_rep_valid && 217 !s0_high_conf_prf_valid && 218 !s0_int_iss_valid && 219 !s0_vec_iss_valid && 220 !s0_l2l_fwd_valid 221 dontTouch(s0_super_ld_rep_ready) 222 dontTouch(s0_ld_fast_rep_ready) 223 dontTouch(s0_ld_rep_ready) 224 dontTouch(s0_high_conf_prf_ready) 225 dontTouch(s0_int_iss_ready) 226 dontTouch(s0_vec_iss_ready) 227 dontTouch(s0_l2l_fwd_ready) 228 dontTouch(s0_low_conf_prf_ready) 229 230 // load flow source select (OH) 231 val s0_super_ld_rep_select = s0_super_ld_rep_valid && s0_super_ld_rep_ready 232 val s0_ld_fast_rep_select = s0_ld_fast_rep_valid && s0_ld_fast_rep_ready 233 val s0_ld_rep_select = s0_ld_rep_valid && s0_ld_rep_ready 234 val s0_hw_prf_select = s0_high_conf_prf_ready && s0_high_conf_prf_valid || 235 s0_low_conf_prf_ready && s0_low_conf_prf_valid 236 val s0_int_iss_select = s0_int_iss_ready && s0_int_iss_valid 237 val s0_vec_iss_select = s0_vec_iss_ready && s0_vec_iss_valid 238 val s0_l2l_fwd_select = s0_l2l_fwd_ready && s0_l2l_fwd_valid 239 assert(!s0_vec_iss_select) // to be added 240 dontTouch(s0_super_ld_rep_select) 241 dontTouch(s0_ld_fast_rep_select) 242 dontTouch(s0_ld_rep_select) 243 dontTouch(s0_hw_prf_select) 244 dontTouch(s0_int_iss_select) 245 dontTouch(s0_vec_iss_select) 246 dontTouch(s0_l2l_fwd_select) 247 248 s0_valid := (s0_super_ld_rep_valid || 249 s0_ld_fast_rep_valid || 250 s0_ld_rep_valid || 251 s0_high_conf_prf_valid || 252 s0_int_iss_valid || 253 s0_vec_iss_valid || 254 s0_l2l_fwd_valid || 255 s0_low_conf_prf_valid) && io.dcache.req.ready && !s0_kill 256 257 // which is S0's out is ready and dcache is ready 258 val s0_try_ptr_chasing = s0_l2l_fwd_select 259 val s0_do_try_ptr_chasing = s0_try_ptr_chasing && s0_can_go && io.dcache.req.ready 260 val s0_ptr_chasing_vaddr = io.ldu_io.l2l_fwd_in.data(5, 0) +& io.ldu_io.ld_fast_imm(5, 0) 261 val s0_ptr_chasing_canceled = WireInit(false.B) 262 s0_kill := s0_ptr_chasing_canceled || (s0_out.uop.robIdx.needFlush(io.redirect) && !s0_try_ptr_chasing) 263 264 // prefetch related ctrl signal 265 val s0_prf = Wire(Bool()) 266 val s0_prf_rd = Wire(Bool()) 267 val s0_prf_wr = Wire(Bool()) 268 val s0_hw_prf = s0_hw_prf_select 269 270 io.ldu_io.canAcceptLowConfPrefetch := s0_low_conf_prf_ready 271 io.ldu_io.canAcceptHighConfPrefetch := s0_high_conf_prf_ready 272 273 // query DTLB 274 io.tlb.req.valid := s0_valid 275 io.tlb.req.bits.cmd := Mux(s0_prf, 276 Mux(s0_prf_wr, TlbCmd.write, TlbCmd.read), 277 Mux(s0_ld_flow, TlbCmd.read, TlbCmd.write) 278 ) 279 io.tlb.req.bits.vaddr := Mux(s0_hw_prf_select, io.ldu_io.prefetch_req.bits.paddr, s0_vaddr) 280 io.tlb.req.bits.size := LSUOpType.size(s0_uop.fuOpType) 281 io.tlb.req.bits.kill := s0_kill 282 io.tlb.req.bits.memidx.is_ld := s0_ld_flow 283 io.tlb.req.bits.memidx.is_st := !s0_ld_flow 284 io.tlb.req.bits.memidx.idx := s0_uop.lqIdx.value 285 io.tlb.req.bits.debug.robIdx := s0_uop.robIdx 286 io.tlb.req.bits.no_translate := s0_hw_prf_select // hw b.reqetch addr does not need to be translated 287 io.tlb.req.bits.debug.pc := s0_uop.pc 288 io.tlb.req.bits.debug.isFirstIssue := s0_isFirstIssue 289 290 // query DCache 291 io.dcache.req.valid := s0_valid 292 io.dcache.req.bits.cmd := Mux(s0_prf_rd, 293 MemoryOpConstants.M_PFR, 294 Mux(s0_prf_wr, MemoryOpConstants.M_PFW, 295 Mux(s0_ld_flow, MemoryOpConstants.M_XRD, MemoryOpConstants.M_XWR)) 296 ) 297 io.dcache.req.bits.vaddr := s0_vaddr 298 io.dcache.req.bits.mask := s0_mask 299 io.dcache.req.bits.data := DontCare 300 io.dcache.req.bits.isFirstIssue := s0_isFirstIssue 301 io.dcache.req.bits.instrtype := Mux(s0_prf, DCACHE_PREFETCH_SOURCE.U, 302 Mux(s0_ld_flow, LOAD_SOURCE.U, STORE_SOURCE.U)) 303 io.dcache.req.bits.debug_robIdx := s0_uop.robIdx.value 304 io.dcache.req.bits.replayCarry := s0_rep_carry 305 io.dcache.req.bits.id := DontCare // TODO: update cache meta 306 io.dcache.pf_source := Mux(s0_hw_prf_select, io.ldu_io.prefetch_req.bits.pf_source.value, L1_HW_PREFETCH_NULL) 307 308 // load flow priority mux 309 def fromNullSource() = { 310 s0_vaddr := 0.U 311 s0_mask := 0.U 312 s0_uop := 0.U.asTypeOf(new DynInst) 313 s0_try_l2l := false.B 314 s0_has_rob_entry := false.B 315 s0_rsIdx := 0.U 316 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 317 s0_mshrid := 0.U 318 s0_isFirstIssue := false.B 319 s0_fast_rep := false.B 320 s0_ld_rep := false.B 321 s0_l2l_fwd := false.B 322 s0_prf := false.B 323 s0_prf_rd := false.B 324 s0_prf_wr := false.B 325 s0_sched_idx := 0.U 326 } 327 328 def fromFastReplaySource(src: LqWriteBundle) = { 329 s0_vaddr := src.vaddr 330 s0_mask := src.mask 331 s0_uop := src.uop 332 s0_try_l2l := false.B 333 s0_has_rob_entry := src.hasROBEntry 334 s0_rep_carry := src.rep_info.rep_carry 335 s0_mshrid := src.rep_info.mshr_id 336 s0_rsIdx := src.rsIdx 337 s0_isFirstIssue := false.B 338 s0_fast_rep := true.B 339 s0_ld_rep := src.isLoadReplay 340 s0_l2l_fwd := false.B 341 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 342 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 343 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 344 s0_sched_idx := src.schedIndex 345 } 346 347 def fromNormalReplaySource(src: LsPipelineBundle) = { 348 s0_vaddr := src.vaddr 349 s0_mask := genVWmask(src.vaddr, src.uop.fuOpType(1, 0)) 350 s0_uop := src.uop 351 s0_try_l2l := false.B 352 s0_has_rob_entry := true.B 353 s0_rsIdx := src.rsIdx 354 s0_rep_carry := src.replayCarry 355 s0_mshrid := src.mshrid 356 s0_isFirstIssue := false.B 357 s0_fast_rep := false.B 358 s0_ld_rep := true.B 359 s0_l2l_fwd := false.B 360 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 361 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 362 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 363 s0_sched_idx := src.schedIndex 364 } 365 366 def fromPrefetchSource(src: L1PrefetchReq) = { 367 s0_vaddr := src.getVaddr() 368 s0_mask := 0.U 369 s0_uop := DontCare 370 s0_try_l2l := false.B 371 s0_has_rob_entry := false.B 372 s0_rsIdx := 0.U 373 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 374 s0_mshrid := 0.U 375 s0_isFirstIssue := false.B 376 s0_fast_rep := false.B 377 s0_ld_rep := false.B 378 s0_l2l_fwd := false.B 379 s0_prf := true.B 380 s0_prf_rd := !src.is_store 381 s0_prf_wr := src.is_store 382 s0_sched_idx := 0.U 383 } 384 385 def fromIntIssueSource(src: MemExuInput) = { 386 s0_vaddr := src.src(0) + SignExt(src.uop.imm(11, 0), VAddrBits) 387 s0_mask := genVWmask(s0_vaddr, src.uop.fuOpType(1,0)) 388 s0_uop := src.uop 389 s0_try_l2l := false.B 390 s0_has_rob_entry := true.B 391 s0_rsIdx := io.rsIdx 392 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 393 s0_mshrid := 0.U 394 s0_isFirstIssue := true.B 395 s0_fast_rep := false.B 396 s0_ld_rep := false.B 397 s0_l2l_fwd := false.B 398 s0_prf := LSUOpType.isPrefetch(src.uop.fuOpType) 399 s0_prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 400 s0_prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 401 s0_sched_idx := 0.U 402 } 403 404 def fromVecIssueSource() = { 405 s0_vaddr := 0.U 406 s0_mask := 0.U 407 s0_uop := 0.U.asTypeOf(new DynInst) 408 s0_try_l2l := false.B 409 s0_has_rob_entry := false.B 410 s0_rsIdx := 0.U 411 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 412 s0_mshrid := 0.U 413 s0_isFirstIssue := false.B 414 s0_fast_rep := false.B 415 s0_ld_rep := false.B 416 s0_l2l_fwd := false.B 417 s0_prf := false.B 418 s0_prf_rd := false.B 419 s0_prf_wr := false.B 420 s0_sched_idx := 0.U 421 } 422 423 def fromLoadToLoadSource(src: LoadToLoadIO) = { 424 s0_vaddr := Cat(src.data(XLEN-1, 6), s0_ptr_chasing_vaddr(5,0)) 425 s0_mask := genVWmask(s0_vaddr, io.ldu_io.ld_fast_fuOpType(1, 0)) 426 // When there's no valid instruction from RS and LSQ, we try the load-to-load forwarding. 427 // Assume the pointer chasing is always ld. 428 s0_uop.fuOpType := io.ldu_io.ld_fast_fuOpType 429 s0_try_l2l := true.B 430 // we dont care s0_isFirstIssue and s0_rsIdx and s0_sqIdx in S0 when trying pointchasing 431 // because these signals will be updated in S1 432 s0_has_rob_entry := false.B 433 s0_rsIdx := 0.U 434 s0_mshrid := 0.U 435 s0_rep_carry := 0.U.asTypeOf(s0_rep_carry.cloneType) 436 s0_isFirstIssue := true.B 437 s0_fast_rep := false.B 438 s0_ld_rep := false.B 439 s0_l2l_fwd := true.B 440 s0_prf := false.B 441 s0_prf_rd := false.B 442 s0_prf_wr := false.B 443 s0_sched_idx := 0.U 444 } 445 446 // set default 447 s0_uop := DontCare 448 when (s0_super_ld_rep_select) { fromNormalReplaySource(io.ldu_io.replay.bits) } 449 .elsewhen (s0_ld_fast_rep_select) { fromFastReplaySource(io.ldu_io.fast_rep_in.bits) } 450 .elsewhen (s0_ld_rep_select) { fromNormalReplaySource(io.ldu_io.replay.bits) } 451 .elsewhen (s0_hw_prf_select) { fromPrefetchSource(io.ldu_io.prefetch_req.bits) } 452 .elsewhen (s0_int_iss_select) { fromIntIssueSource(io.lsin.bits) } 453 .elsewhen (s0_vec_iss_select) { fromVecIssueSource() } 454 .otherwise { 455 if (EnableLoadToLoadForward) { 456 fromLoadToLoadSource(io.ldu_io.l2l_fwd_in) 457 } else { 458 fromNullSource() 459 } 460 } 461 462 // address align check 463 val s0_addr_aligned = LookupTree(s0_uop.fuOpType(1, 0), List( 464 "b00".U -> true.B, //b 465 "b01".U -> (s0_vaddr(0) === 0.U), //h 466 "b10".U -> (s0_vaddr(1, 0) === 0.U), //w 467 "b11".U -> (s0_vaddr(2, 0) === 0.U) //d 468 )) 469 470 // accept load flow if dcache ready (tlb is always ready) 471 // TODO: prefetch need writeback to loadQueueFlag 472 s0_out := DontCare 473 s0_out.rsIdx := s0_rsIdx 474 s0_out.vaddr := s0_vaddr 475 s0_out.mask := s0_mask 476 s0_out.uop := s0_uop 477 s0_out.isFirstIssue := s0_isFirstIssue 478 s0_out.hasROBEntry := s0_has_rob_entry 479 s0_out.isPrefetch := s0_prf 480 s0_out.isHWPrefetch := s0_hw_prf 481 s0_out.isFastReplay := s0_fast_rep 482 s0_out.isLoadReplay := s0_ld_rep 483 s0_out.isFastPath := s0_l2l_fwd 484 s0_out.mshrid := s0_mshrid 485 s0_out.uop.exceptionVec(loadAddrMisaligned) := !s0_addr_aligned && s0_ld_flow 486 s0_out.uop.exceptionVec(storeAddrMisaligned) := !s0_addr_aligned && !s0_ld_flow 487 s0_out.forward_tlDchannel := s0_super_ld_rep_select 488 when(io.tlb.req.valid && s0_isFirstIssue) { 489 s0_out.uop.debugInfo.tlbFirstReqTime := GTimer() 490 }.otherwise{ 491 s0_out.uop.debugInfo.tlbFirstReqTime := s0_uop.debugInfo.tlbFirstReqTime 492 } 493 s0_out.schedIndex := s0_sched_idx 494 495 // load fast replay 496 io.ldu_io.fast_rep_in.ready := (s0_can_go && io.dcache.req.ready && s0_ld_fast_rep_ready) 497 498 // load flow source ready 499 // cache missed load has highest priority 500 // always accept cache missed load flow from load replay queue 501 io.ldu_io.replay.ready := (s0_can_go && io.dcache.req.ready && (s0_ld_rep_ready && !s0_rep_stall || s0_super_ld_rep_select)) 502 503 // accept load flow from rs when: 504 // 1) there is no lsq-replayed load 505 // 2) there is no fast replayed load 506 // 3) there is no high confidence prefetch request 507 io.lsin.ready := (s0_can_go && (io.dcache.req.ready || !s0_ld_flow) && s0_int_iss_ready) 508 509 // for hw prefetch load flow feedback, to be added later 510 // io.prefetch_in.ready := s0_hw_prf_select 511 512 // dcache replacement extra info 513 // TODO: should prefetch load update replacement? 514 io.dcache.replacementUpdated := Mux(s0_ld_rep_select || s0_super_ld_rep_select, io.ldu_io.replay.bits.replacementUpdated, false.B) 515 516 517 io.stu_io.st_mask_out.valid := s0_valid && !s0_ld_flow 518 io.stu_io.st_mask_out.bits.mask := s0_out.mask 519 io.stu_io.st_mask_out.bits.sqIdx := s0_out.uop.sqIdx 520 521 // load debug 522 XSDebug(io.dcache.req.fire && s0_ld_flow, 523 p"[DCACHE LOAD REQ] pc ${Hexadecimal(s0_uop.pc)}, vaddr ${Hexadecimal(s0_vaddr)}\n" 524 ) 525 XSDebug(s0_valid && s0_ld_flow, 526 p"S0: pc ${Hexadecimal(s0_out.uop.pc)}, lqIdx ${Hexadecimal(s0_out.uop.lqIdx.asUInt)}, " + 527 p"vaddr ${Hexadecimal(s0_out.vaddr)}, mask ${Hexadecimal(s0_out.mask)}\n") 528 529 // store debug 530 XSDebug(io.dcache.req.fire && !s0_ld_flow, 531 p"[DCACHE STORE REQ] pc ${Hexadecimal(s0_uop.pc)}, vaddr ${Hexadecimal(s0_vaddr)}\n" 532 ) 533 XSDebug(s0_valid && !s0_ld_flow, 534 p"S0: pc ${Hexadecimal(s0_out.uop.pc)}, sqIdx ${Hexadecimal(s0_out.uop.sqIdx.asUInt)}, " + 535 p"vaddr ${Hexadecimal(s0_out.vaddr)}, mask ${Hexadecimal(s0_out.mask)}\n") 536 537 538 // Pipeline 539 // -------------------------------------------------------------------------------- 540 // stage 1 541 // -------------------------------------------------------------------------------- 542 // TLB resp (send paddr to dcache) 543 val s1_valid = RegInit(false.B) 544 val s1_in = Wire(new LqWriteBundle) 545 val s1_out = Wire(new LqWriteBundle) 546 val s1_kill = Wire(Bool()) 547 val s1_can_go = s2_ready 548 val s1_fire = s1_valid && !s1_kill && s1_can_go 549 val s1_ld_flow = RegNext(s0_ld_flow) 550 551 s1_ready := !s1_valid || s1_kill || s2_ready 552 when (s0_fire) { s1_valid := true.B } 553 .elsewhen (s1_fire) { s1_valid := false.B } 554 .elsewhen (s1_kill) { s1_valid := false.B } 555 s1_in := RegEnable(s0_out, s0_fire) 556 557 val s1_fast_rep_dly_err = RegNext(io.ldu_io.fast_rep_in.bits.delayedLoadError) 558 val s1_fast_rep_kill = s1_fast_rep_dly_err && s1_in.isFastReplay 559 val s1_l2l_fwd_dly_err = RegNext(io.ldu_io.l2l_fwd_in.dly_ld_err) 560 val s1_l2l_fwd_kill = s1_l2l_fwd_dly_err && s1_in.isFastPath 561 val s1_late_kill = s1_fast_rep_kill || s1_l2l_fwd_kill 562 val s1_vaddr_hi = Wire(UInt()) 563 val s1_vaddr_lo = Wire(UInt()) 564 val s1_vaddr = Wire(UInt()) 565 val s1_paddr_dup_lsu = Wire(UInt()) 566 val s1_paddr_dup_dcache = Wire(UInt()) 567 val s1_ld_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, LduCfg).asUInt.orR // af & pf exception were modified below. 568 val s1_st_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, StaCfg).asUInt.orR // af & pf exception were modified below. 569 val s1_exception = (s1_ld_flow && s1_ld_exception) || (!s1_ld_flow && s1_st_exception) 570 val s1_tlb_miss = io.tlb.resp.bits.miss 571 val s1_prf = s1_in.isPrefetch 572 val s1_hw_prf = s1_in.isHWPrefetch 573 val s1_sw_prf = s1_prf && !s1_hw_prf 574 val s1_tlb_memidx = io.tlb.resp.bits.memidx 575 576 s1_vaddr_hi := s1_in.vaddr(VAddrBits - 1, 6) 577 s1_vaddr_lo := s1_in.vaddr(5, 0) 578 s1_vaddr := Cat(s1_vaddr_hi, s1_vaddr_lo) 579 s1_paddr_dup_lsu := io.tlb.resp.bits.paddr(0) 580 s1_paddr_dup_dcache := io.tlb.resp.bits.paddr(1) 581 582 when (s1_tlb_memidx.is_ld && io.tlb.resp.valid && !s1_tlb_miss && 583 s1_tlb_memidx.idx === s1_in.uop.lqIdx.value && s1_ld_flow) { 584 // printf("Load idx = %d\n", s1_tlb_memidx.idx) 585 s1_out.uop.debugInfo.tlbRespTime := GTimer() 586 } .elsewhen(s1_tlb_memidx.is_st && io.tlb.resp.valid && !s1_tlb_miss && 587 s1_tlb_memidx.idx === s1_out.uop.sqIdx.value && !s1_ld_flow) { 588 // printf("Store idx = %d\n", s1_tlb_memidx.idx) 589 s1_out.uop.debugInfo.tlbRespTime := GTimer() 590 } 591 592 io.tlb.req_kill := s1_kill 593 io.tlb.resp.ready := true.B 594 595 io.dcache.s1_paddr_dup_lsu <> s1_paddr_dup_lsu 596 io.dcache.s1_paddr_dup_dcache <> s1_paddr_dup_dcache 597 io.dcache.s1_kill := s1_kill || s1_tlb_miss || s1_exception || !s1_ld_flow 598 599 // store to load forwarding 600 io.ldu_io.sbuffer.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_fast_rep_kill || s1_prf || !s1_ld_flow) 601 io.ldu_io.sbuffer.vaddr := s1_vaddr 602 io.ldu_io.sbuffer.paddr := s1_paddr_dup_lsu 603 io.ldu_io.sbuffer.uop := s1_in.uop 604 io.ldu_io.sbuffer.sqIdx := s1_in.uop.sqIdx 605 io.ldu_io.sbuffer.mask := s1_in.mask 606 io.ldu_io.sbuffer.pc := s1_in.uop.pc // FIXME: remove it 607 608 io.ldu_io.lsq.forward.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_fast_rep_kill || s1_prf || !s1_ld_flow) 609 io.ldu_io.lsq.forward.vaddr := s1_vaddr 610 io.ldu_io.lsq.forward.paddr := s1_paddr_dup_lsu 611 io.ldu_io.lsq.forward.uop := s1_in.uop 612 io.ldu_io.lsq.forward.sqIdx := s1_in.uop.sqIdx 613 io.ldu_io.lsq.forward.sqIdxMask := 0.U 614 io.ldu_io.lsq.forward.mask := s1_in.mask 615 io.ldu_io.lsq.forward.pc := s1_in.uop.pc // FIXME: remove it 616 617 // st-ld violation query 618 val s1_nuke = VecInit((0 until StorePipelineWidth).map(w => { 619 io.ldu_io.stld_nuke_query(w).valid && // query valid 620 isAfter(s1_in.uop.robIdx, io.ldu_io.stld_nuke_query(w).bits.robIdx) && // older store 621 // TODO: Fix me when vector instruction 622 (s1_paddr_dup_lsu(PAddrBits-1, 3) === io.ldu_io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 623 (s1_in.mask & io.ldu_io.stld_nuke_query(w).bits.mask).orR // data mask contain 624 })).asUInt.orR && !s1_tlb_miss && s1_ld_flow 625 626 s1_out := s1_in 627 s1_out.vaddr := s1_vaddr 628 s1_out.paddr := s1_paddr_dup_lsu 629 s1_out.tlbMiss := s1_tlb_miss 630 s1_out.ptwBack := io.tlb.resp.bits.ptwBack 631 s1_out.rsIdx := s1_in.rsIdx 632 s1_out.rep_info.debug := s1_in.uop.debugInfo 633 s1_out.rep_info.nuke := s1_nuke && !s1_sw_prf 634 s1_out.lateKill := s1_late_kill 635 636 when (s1_ld_flow) { 637 when (!s1_late_kill) { 638 // current ori test will cause the case of ldest == 0, below will be modifeid in the future. 639 // af & pf exception were modified 640 s1_out.uop.exceptionVec(loadPageFault) := io.tlb.resp.bits.excp(0).pf.ld 641 s1_out.uop.exceptionVec(loadAccessFault) := io.tlb.resp.bits.excp(0).af.ld 642 } .otherwise { 643 s1_out.uop.exceptionVec(loadAddrMisaligned) := false.B 644 s1_out.uop.exceptionVec(loadAccessFault) := s1_late_kill 645 } 646 } .otherwise { 647 s1_out.uop.exceptionVec(storePageFault) := io.tlb.resp.bits.excp(0).pf.st 648 s1_out.uop.exceptionVec(storeAccessFault) := io.tlb.resp.bits.excp(0).af.st 649 } 650 651 // pointer chasing 652 val s1_try_ptr_chasing = RegNext(s0_do_try_ptr_chasing, false.B) 653 val s1_ptr_chasing_vaddr = RegEnable(s0_ptr_chasing_vaddr, s0_do_try_ptr_chasing) 654 val s1_fu_op_type_not_ld = WireInit(false.B) 655 val s1_not_fast_match = WireInit(false.B) 656 val s1_addr_mismatch = WireInit(false.B) 657 val s1_addr_misaligned = WireInit(false.B) 658 val s1_ptr_chasing_canceled = WireInit(false.B) 659 val s1_cancel_ptr_chasing = WireInit(false.B) 660 661 s1_kill := s1_late_kill || 662 s1_cancel_ptr_chasing || 663 s1_in.uop.robIdx.needFlush(io.redirect) || 664 RegEnable(s0_kill, false.B, io.lsin.valid || io.ldu_io.replay.valid || io.ldu_io.l2l_fwd_in.valid || io.ldu_io.fast_rep_in.valid) 665 666 if (EnableLoadToLoadForward) { 667 // Sometimes, we need to cancel the load-load forwarding. 668 // These can be put at S0 if timing is bad at S1. 669 // Case 0: CACHE_SET(base + offset) != CACHE_SET(base) (lowest 6-bit addition has an overflow) 670 s1_addr_mismatch := s1_ptr_chasing_vaddr(6) || RegEnable(io.ldu_io.ld_fast_imm(11, 6).orR, s0_do_try_ptr_chasing) 671 // Case 1: the address is misaligned, kill s1 672 s1_addr_misaligned := LookupTree(s1_in.uop.fuOpType(1, 0), List( 673 "b00".U -> false.B, //b 674 "b01".U -> (s1_vaddr(0) =/= 0.U), //h 675 "b10".U -> (s1_vaddr(1, 0) =/= 0.U), //w 676 "b11".U -> (s1_vaddr(2, 0) =/= 0.U) //d 677 )) 678 // Case 2: this load-load uop is cancelled 679 s1_ptr_chasing_canceled := !io.lsin.valid 680 681 when (s1_try_ptr_chasing) { 682 s1_cancel_ptr_chasing := s1_addr_mismatch || s1_addr_misaligned || s1_ptr_chasing_canceled 683 684 s1_in.uop := io.lsin.bits.uop 685 s1_in.rsIdx := io.rsIdx 686 s1_in.isFirstIssue := io.isFirstIssue 687 s1_vaddr_lo := s1_ptr_chasing_vaddr(5, 0) 688 s1_paddr_dup_lsu := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 689 s1_paddr_dup_dcache := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 690 691 // recored tlb time when get the data to ensure the correctness of the latency calculation (although it should not record in here, because it does not use tlb) 692 s1_in.uop.debugInfo.tlbFirstReqTime := GTimer() 693 s1_in.uop.debugInfo.tlbRespTime := GTimer() 694 } 695 when (!s1_cancel_ptr_chasing) { 696 s0_ptr_chasing_canceled := s1_try_ptr_chasing && !io.ldu_io.replay.fire && !io.ldu_io.fast_rep_in.fire 697 when (s1_try_ptr_chasing) { 698 io.lsin.ready := true.B 699 } 700 } 701 } 702 703 // pre-calcuate sqIdx mask in s0, then send it to lsq in s1 for forwarding 704 val s1_sqIdx_mask = RegNext(UIntToMask(s0_out.uop.sqIdx.value, StoreQueueSize)) 705 // to enable load-load, sqIdxMask must be calculated based on lsin.uop 706 // If the timing here is not OK, load-load forwarding has to be disabled. 707 // Or we calculate sqIdxMask at RS?? 708 io.ldu_io.lsq.forward.sqIdxMask := s1_sqIdx_mask 709 if (EnableLoadToLoadForward) { 710 when (s1_try_ptr_chasing) { 711 io.ldu_io.lsq.forward.sqIdxMask := UIntToMask(io.lsin.bits.uop.sqIdx.value, StoreQueueSize) 712 } 713 } 714 715 io.ldu_io.forward_mshr.valid := s1_valid && s1_out.forward_tlDchannel && s1_ld_flow 716 io.ldu_io.forward_mshr.mshrid := s1_out.mshrid 717 io.ldu_io.forward_mshr.paddr := s1_out.paddr 718 719 720 // load debug 721 XSDebug(s1_valid && s1_ld_flow, 722 p"S1: pc ${Hexadecimal(s1_out.uop.pc)}, lId ${Hexadecimal(s1_out.uop.lqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 723 p"paddr ${Hexadecimal(s1_out.paddr)}, mmio ${s1_out.mmio}\n") 724 725 // store debug 726 XSDebug(s1_valid && !s1_ld_flow, 727 p"S1: pc ${Hexadecimal(s1_out.uop.pc)}, lId ${Hexadecimal(s1_out.uop.sqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 728 p"paddr ${Hexadecimal(s1_out.paddr)}, mmio ${s1_out.mmio}\n") 729 730 // store out 731 io.stu_io.lsq.valid := s1_valid && !s1_ld_flow 732 io.stu_io.lsq.bits := s1_out 733 io.stu_io.lsq.bits.miss := s1_tlb_miss 734 735 // st-ld violation dectect request 736 io.stu_io.stld_nuke_query.valid := s1_valid && !s1_tlb_miss && !s1_ld_flow 737 io.stu_io.stld_nuke_query.bits.robIdx := s1_in.uop.robIdx 738 io.stu_io.stld_nuke_query.bits.paddr := s1_paddr_dup_lsu 739 io.stu_io.stld_nuke_query.bits.mask := s1_in.mask 740 741 // Pipeline 742 // -------------------------------------------------------------------------------- 743 // stage 2 744 // -------------------------------------------------------------------------------- 745 // s2: DCache resp 746 val s2_valid = RegInit(false.B) 747 val s2_in = Wire(new LqWriteBundle) 748 val s2_out = Wire(new LqWriteBundle) 749 val s2_kill = Wire(Bool()) 750 val s2_can_go = s3_ready 751 val s2_fire = s2_valid && !s2_kill && s2_can_go 752 753 s2_kill := s2_in.uop.robIdx.needFlush(io.redirect) 754 s2_ready := !s2_valid || s2_kill || s3_ready 755 when (s1_fire) { s2_valid := true.B } 756 .elsewhen (s2_fire) { s2_valid := false.B } 757 .elsewhen (s2_kill) { s2_valid := false.B } 758 s2_in := RegEnable(s1_out, s1_fire) 759 760 val s2_pmp = WireInit(io.pmp) 761 762 val s2_prf = s2_in.isPrefetch 763 val s2_hw_prf = s2_in.isHWPrefetch 764 val s2_ld_flow = RegEnable(s1_ld_flow, s1_fire) 765 766 // exception that may cause load addr to be invalid / illegal 767 // if such exception happen, that inst and its exception info 768 // will be force writebacked to rob 769 val s2_exception_vec = WireInit(s2_in.uop.exceptionVec) 770 when (s2_ld_flow) { 771 when (!s2_in.lateKill) { 772 s2_exception_vec(loadAccessFault) := s2_in.uop.exceptionVec(loadAccessFault) || s2_pmp.ld 773 // soft prefetch will not trigger any exception (but ecc error interrupt may be triggered) 774 when (s2_prf || s2_in.tlbMiss) { 775 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 776 } 777 } 778 } .otherwise { 779 s2_exception_vec(storeAccessFault) := s2_in.uop.exceptionVec(storeAccessFault) || s2_pmp.st 780 when (s2_prf || s2_in.tlbMiss) { 781 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 782 } 783 } 784 val s2_ld_exception = ExceptionNO.selectByFu(s2_exception_vec, LduCfg).asUInt.orR && s2_ld_flow 785 val s2_st_exception = ExceptionNO.selectByFu(s2_exception_vec, StaCfg).asUInt.orR && !s2_ld_flow 786 val s2_exception = s2_ld_exception || s2_st_exception 787 788 val (s2_fwd_frm_d_chan, s2_fwd_data_frm_d_chan) = io.ldu_io.tl_d_channel.forward(s1_valid && s1_out.forward_tlDchannel, s1_out.mshrid, s1_out.paddr) 789 val (s2_fwd_data_valid, s2_fwd_frm_mshr, s2_fwd_data_frm_mshr) = io.ldu_io.forward_mshr.forward() 790 val s2_fwd_frm_d_chan_or_mshr = s2_fwd_data_valid && (s2_fwd_frm_d_chan || s2_fwd_frm_mshr) 791 792 // writeback access fault caused by ecc error / bus error 793 // * ecc data error is slow to generate, so we will not use it until load stage 3 794 // * in load stage 3, an extra signal io.load_error will be used to 795 val s2_actually_mmio = s2_pmp.mmio 796 val s2_mmio = !s2_prf && 797 s2_actually_mmio && 798 !s2_exception && 799 !s2_in.tlbMiss 800 val s2_full_fwd = Wire(Bool()) 801 val s2_mem_amb = s2_in.uop.storeSetHit && 802 io.ldu_io.lsq.forward.addrInvalid 803 804 val s2_tlb_miss = s2_in.tlbMiss 805 val s2_fwd_fail = io.ldu_io.lsq.forward.dataInvalid 806 val s2_dcache_miss = io.dcache.resp.bits.miss && 807 !s2_fwd_frm_d_chan_or_mshr && 808 !s2_full_fwd 809 810 val s2_mq_nack = io.dcache.s2_mq_nack && 811 !s2_fwd_frm_d_chan_or_mshr && 812 !s2_full_fwd 813 814 val s2_bank_conflict = io.dcache.s2_bank_conflict && 815 !s2_fwd_frm_d_chan_or_mshr && 816 !s2_full_fwd 817 818 val s2_wpu_pred_fail = io.dcache.s2_wpu_pred_fail && 819 !s2_fwd_frm_d_chan_or_mshr && 820 !s2_full_fwd 821 822 val s2_rar_nack = io.ldu_io.lsq.ldld_nuke_query.req.valid && 823 !io.ldu_io.lsq.ldld_nuke_query.req.ready 824 825 val s2_raw_nack = io.ldu_io.lsq.stld_nuke_query.req.valid && 826 !io.ldu_io.lsq.stld_nuke_query.req.ready 827 828 // st-ld violation query 829 // NeedFastRecovery Valid when 830 // 1. Fast recovery query request Valid. 831 // 2. Load instruction is younger than requestors(store instructions). 832 // 3. Physical address match. 833 // 4. Data contains. 834 val s2_nuke = VecInit((0 until StorePipelineWidth).map(w => { 835 io.ldu_io.stld_nuke_query(w).valid && // query valid 836 isAfter(s2_in.uop.robIdx, io.ldu_io.stld_nuke_query(w).bits.robIdx) && // older store 837 // TODO: Fix me when vector instruction 838 (s2_in.paddr(PAddrBits-1, 3) === io.ldu_io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3)) && // paddr match 839 (s2_in.mask & io.ldu_io.stld_nuke_query(w).bits.mask).orR // data mask contain 840 })).asUInt.orR && s2_ld_flow || s2_in.rep_info.nuke 841 842 val s2_cache_handled = io.dcache.resp.bits.handled 843 val s2_cache_tag_error = RegNext(io.csrCtrl.cache_error_enable) && 844 io.dcache.resp.bits.tag_error 845 846 val s2_troublem = !s2_exception && 847 !s2_mmio && 848 !s2_prf && 849 !s2_in.lateKill 850 851 io.dcache.resp.ready := true.B 852 val s2_dcache_should_resp = !(s2_in.tlbMiss || s2_exception || s2_mmio || s2_prf || s2_in.lateKill) && s2_ld_flow 853 assert(!(s2_valid && (s2_dcache_should_resp && !io.dcache.resp.valid)), "DCache response got lost") 854 855 // fast replay require 856 val s2_dcache_fast_rep = (s2_mq_nack || !s2_dcache_miss && (s2_bank_conflict || s2_wpu_pred_fail)) 857 val s2_nuke_fast_rep = !s2_mq_nack && 858 !s2_dcache_miss && 859 !s2_bank_conflict && 860 !s2_wpu_pred_fail && 861 !s2_rar_nack && 862 !s2_raw_nack && 863 s2_nuke 864 865 val s2_fast_rep = !s2_mem_amb && 866 !s2_tlb_miss && 867 !s2_fwd_fail && 868 (s2_dcache_fast_rep || s2_nuke_fast_rep) && 869 s2_troublem 870 871 // need allocate new entry 872 val s2_can_query = !s2_mem_amb && 873 !s2_tlb_miss && 874 !s2_fwd_fail && 875 !s2_dcache_fast_rep && 876 s2_troublem 877 878 val s2_data_fwded = s2_dcache_miss && (s2_full_fwd || s2_cache_tag_error) 879 880 // ld-ld violation require 881 io.ldu_io.lsq.ldld_nuke_query.req.valid := s2_valid && s2_can_query 882 io.ldu_io.lsq.ldld_nuke_query.req.bits.uop := s2_in.uop 883 io.ldu_io.lsq.ldld_nuke_query.req.bits.mask := s2_in.mask 884 io.ldu_io.lsq.ldld_nuke_query.req.bits.paddr := s2_in.paddr 885 io.ldu_io.lsq.ldld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid, true.B, !s2_dcache_miss) 886 887 // st-ld violation require 888 io.ldu_io.lsq.stld_nuke_query.req.valid := s2_valid && s2_can_query 889 io.ldu_io.lsq.stld_nuke_query.req.bits.uop := s2_in.uop 890 io.ldu_io.lsq.stld_nuke_query.req.bits.mask := s2_in.mask 891 io.ldu_io.lsq.stld_nuke_query.req.bits.paddr := s2_in.paddr 892 io.ldu_io.lsq.stld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid, true.B, !s2_dcache_miss) 893 894 // merge forward result 895 // lsq has higher priority than sbuffer 896 val s2_fwd_mask = Wire(Vec((VLEN/8), Bool())) 897 val s2_fwd_data = Wire(Vec((VLEN/8), UInt(8.W))) 898 s2_full_fwd := ((~s2_fwd_mask.asUInt).asUInt & s2_in.mask) === 0.U && !io.ldu_io.lsq.forward.dataInvalid 899 // generate XLEN/8 Muxs 900 for (i <- 0 until VLEN / 8) { 901 s2_fwd_mask(i) := io.ldu_io.lsq.forward.forwardMask(i) || io.ldu_io.sbuffer.forwardMask(i) 902 s2_fwd_data(i) := Mux(io.ldu_io.lsq.forward.forwardMask(i), io.ldu_io.lsq.forward.forwardData(i), io.ldu_io.sbuffer.forwardData(i)) 903 } 904 905 XSDebug(s2_fire && s2_ld_flow, "[FWD LOAD RESP] pc %x fwd %x(%b) + %x(%b)\n", 906 s2_in.uop.pc, 907 io.ldu_io.lsq.forward.forwardData.asUInt, io.ldu_io.lsq.forward.forwardMask.asUInt, 908 s2_in.forwardData.asUInt, s2_in.forwardMask.asUInt 909 ) 910 911 // 912 s2_out := s2_in 913 s2_out.data := 0.U // data will be generated in load s3 914 s2_out.uop.fpWen := s2_in.uop.fpWen && !s2_exception && s2_ld_flow 915 s2_out.mmio := s2_mmio 916 s2_out.atomic := s2_pmp.atomic && !s2_ld_flow 917 s2_out.uop.flushPipe := false.B 918 s2_out.uop.exceptionVec := s2_exception_vec 919 s2_out.forwardMask := s2_fwd_mask 920 s2_out.forwardData := s2_fwd_data 921 s2_out.handledByMSHR := s2_cache_handled 922 s2_out.miss := s2_dcache_miss && s2_troublem 923 s2_out.uop.fpWen := s2_in.uop.fpWen && !s2_exception && s2_ld_flow 924 925 // Generate replay signal caused by: 926 // * st-ld violation check 927 // * tlb miss 928 // * dcache replay 929 // * forward data invalid 930 // * dcache miss 931 s2_out.rep_info.mem_amb := s2_mem_amb && s2_troublem 932 s2_out.rep_info.tlb_miss := s2_tlb_miss && s2_troublem 933 s2_out.rep_info.fwd_fail := s2_fwd_fail && s2_troublem 934 s2_out.rep_info.dcache_rep := s2_mq_nack && s2_troublem 935 s2_out.rep_info.dcache_miss := s2_dcache_miss && s2_troublem 936 s2_out.rep_info.bank_conflict := s2_bank_conflict && s2_troublem 937 s2_out.rep_info.wpu_fail := s2_wpu_pred_fail && s2_troublem 938 s2_out.rep_info.rar_nack := s2_rar_nack && s2_troublem 939 s2_out.rep_info.raw_nack := s2_raw_nack && s2_troublem 940 s2_out.rep_info.nuke := s2_nuke && s2_troublem 941 s2_out.rep_info.full_fwd := s2_data_fwded 942 s2_out.rep_info.data_inv_sq_idx := io.ldu_io.lsq.forward.dataInvalidSqIdx 943 s2_out.rep_info.addr_inv_sq_idx := io.ldu_io.lsq.forward.addrInvalidSqIdx 944 s2_out.rep_info.rep_carry := io.dcache.resp.bits.replayCarry 945 s2_out.rep_info.mshr_id := io.dcache.resp.bits.mshr_id 946 s2_out.rep_info.last_beat := s2_in.paddr(log2Up(refillBytes)) 947 s2_out.rep_info.debug := s2_in.uop.debugInfo 948 949 // if forward fail, replay this inst from fetch 950 val debug_fwd_fail_rep = s2_fwd_fail && !s2_troublem && !s2_in.tlbMiss 951 // if ld-ld violation is detected, replay from this inst from fetch 952 val debug_ldld_nuke_rep = false.B // s2_ldld_violation && !s2_mmio && !s2_is_prefetch && !s2_in.tlbMiss 953 // io.out.bits.uop.replayInst := false.B 954 955 // to be removed 956 val s2_ld_need_fb = !s2_in.isLoadReplay && // already feedbacked 957 io.ldu_io.lq_rep_full && // LoadQueueReplay is full 958 s2_out.rep_info.need_rep && // need replay 959 !s2_exception && // no exception is triggered 960 !s2_hw_prf // not hardware prefetch 961 val s2_st_need_fb = !s2_ld_flow 962 io.feedback_fast.valid := s2_valid && (s2_ld_need_fb || s2_st_need_fb) 963 io.feedback_fast.bits.hit := false.B 964 io.feedback_fast.bits.flushState := s2_in.ptwBack 965 io.feedback_fast.bits.robIdx := s2_in.uop.robIdx 966 io.feedback_fast.bits.sourceType := Mux(s2_ld_flow, RSFeedbackType.lrqFull, RSFeedbackType.tlbMiss) 967 io.feedback_fast.bits.dataInvalidSqIdx := DontCare 968 969 io.ldu_io.ldCancel.ld1Cancel.valid := s2_valid && ( 970 (s2_out.rep_info.need_rep && s2_out.isFirstIssue) || // exe fail and issued from IQ 971 s2_mmio // is mmio 972 ) 973 io.ldu_io.ldCancel.ld1Cancel.bits := s2_out.deqPortIdx 974 975 // fast wakeup 976 io.ldu_io.fast_uop.valid := RegNext( 977 !io.dcache.s1_disable_fast_wakeup && 978 s1_valid && 979 !s1_kill && 980 !io.tlb.resp.bits.miss && 981 !io.ldu_io.lsq.forward.dataInvalidFast 982 ) && (s2_valid && !s2_out.rep_info.need_rep && !s2_mmio && s2_ld_flow) 983 io.ldu_io.fast_uop.bits := RegNext(s1_out.uop) 984 985 // 986 io.ldu_io.s2_ptr_chasing := RegEnable(s1_try_ptr_chasing && !s1_cancel_ptr_chasing, false.B, s1_fire) 987 988 // prefetch train 989 io.ldu_io.prefetch_train.valid := s2_valid && !s2_actually_mmio && !s2_in.tlbMiss 990 io.ldu_io.prefetch_train.bits.fromLsPipelineBundle(s2_in) 991 io.ldu_io.prefetch_train.bits.miss := io.dcache.resp.bits.miss // TODO: use trace with bank conflict? 992 io.ldu_io.prefetch_train.bits.meta_prefetch := io.dcache.resp.bits.meta_prefetch 993 io.ldu_io.prefetch_train.bits.meta_access := io.dcache.resp.bits.meta_access 994 995 io.ldu_io.prefetch_train_l1.valid := s2_valid && !s2_actually_mmio 996 io.ldu_io.prefetch_train_l1.bits.fromLsPipelineBundle(s2_in) 997 io.ldu_io.prefetch_train_l1.bits.miss := io.dcache.resp.bits.miss 998 io.ldu_io.prefetch_train_l1.bits.meta_prefetch := io.dcache.resp.bits.meta_prefetch 999 io.ldu_io.prefetch_train_l1.bits.meta_access := io.dcache.resp.bits.meta_access 1000 if (env.FPGAPlatform){ 1001 io.dcache.s0_pc := DontCare 1002 io.dcache.s1_pc := DontCare 1003 io.dcache.s2_pc := DontCare 1004 }else{ 1005 io.dcache.s0_pc := s0_out.uop.pc 1006 io.dcache.s1_pc := s1_out.uop.pc 1007 io.dcache.s2_pc := s2_out.uop.pc 1008 } 1009 io.dcache.s2_kill := s2_pmp.ld || s2_pmp.st || s2_actually_mmio || s2_kill || !s2_ld_flow 1010 1011 val s1_ld_left_fire = s1_valid && !s1_kill && s2_ready && s1_ld_flow 1012 val s2_ld_valid_dup = RegInit(0.U(6.W)) 1013 s2_ld_valid_dup := 0x0.U(6.W) 1014 when (s1_ld_left_fire && !s1_out.isHWPrefetch) { s2_ld_valid_dup := 0x3f.U(6.W) } 1015 when (s1_kill || s1_out.isHWPrefetch) { s2_ld_valid_dup := 0x0.U(6.W) } 1016 assert(RegNext((s2_valid === s2_ld_valid_dup(0)) || RegNext(s1_out.isHWPrefetch))) 1017 1018 // Pipeline 1019 // -------------------------------------------------------------------------------- 1020 // stage 3 1021 // -------------------------------------------------------------------------------- 1022 // writeback and update load queue 1023 val s3_valid = RegNext(s2_valid && !s2_out.isHWPrefetch && !s2_out.uop.robIdx.needFlush(io.redirect)) 1024 val s3_in = RegEnable(s2_out, s2_fire) 1025 val s3_out = Wire(Valid(new MemExuOutput)) 1026 val s3_dcache_rep = RegEnable(s2_dcache_fast_rep && s2_troublem, false.B, s2_fire) 1027 val s3_ld_valid_dup = RegEnable(s2_ld_valid_dup, s2_fire) 1028 val s3_fast_rep = Wire(Bool()) 1029 val s3_ld_flow = RegNext(s2_ld_flow) 1030 val s3_troublem = RegNext(s2_troublem) 1031 val s3_kill = s3_in.uop.robIdx.needFlush(io.redirect) 1032 s3_ready := !s3_valid || s3_kill || sx_can_go 1033 1034 // forwrad last beat 1035 val (s3_fwd_frm_d_chan, s3_fwd_data_frm_d_chan) = io.ldu_io.tl_d_channel.forward(s2_valid && s2_out.forward_tlDchannel, s2_out.mshrid, s2_out.paddr) 1036 val s3_fwd_data_valid = RegEnable(s2_fwd_data_valid, false.B, s2_valid) 1037 val s3_fwd_frm_d_chan_valid = (s3_fwd_frm_d_chan && s3_fwd_data_valid) && s3_ld_flow 1038 1039 1040 // s3 load fast replay 1041 io.ldu_io.fast_rep_out.valid := s3_valid && 1042 s3_fast_rep && 1043 !s3_in.uop.robIdx.needFlush(io.redirect) && 1044 s3_ld_flow 1045 io.ldu_io.fast_rep_out.bits := s3_in 1046 1047 io.ldu_io.lsq.ldin.valid := s3_valid && 1048 (!s3_fast_rep || !io.ldu_io.fast_rep_out.ready) && 1049 !s3_in.feedbacked && 1050 !s3_in.lateKill && 1051 s3_ld_flow 1052 io.ldu_io.lsq.ldin.bits := s3_in 1053 io.ldu_io.lsq.ldin.bits.miss := s3_in.miss && !s3_fwd_frm_d_chan_valid 1054 1055 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1056 io.ldu_io.lsq.ldin.bits.data_wen_dup := s3_ld_valid_dup.asBools 1057 io.ldu_io.lsq.ldin.bits.replacementUpdated := io.dcache.resp.bits.replacementUpdated 1058 io.ldu_io.lsq.ldin.bits.missDbUpdated := RegNext(s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated) 1059 1060 val s3_dly_ld_err = 1061 if (EnableAccurateLoadError) { 1062 (s3_in.lateKill || io.dcache.resp.bits.error_delayed) && RegNext(io.csrCtrl.cache_error_enable) 1063 } else { 1064 WireInit(false.B) 1065 } 1066 io.ldu_io.s3_dly_ld_err := false.B // s3_dly_ld_err && s3_valid 1067 io.ldu_io.fast_rep_out.bits.delayedLoadError := s3_dly_ld_err 1068 io.ldu_io.lsq.ldin.bits.dcacheRequireReplay := s3_dcache_rep 1069 1070 val s3_vp_match_fail = RegNext(io.ldu_io.lsq.forward.matchInvalid || io.ldu_io.sbuffer.matchInvalid) && s3_troublem 1071 val s3_ldld_rep_inst = 1072 io.ldu_io.lsq.ldld_nuke_query.resp.valid && 1073 io.ldu_io.lsq.ldld_nuke_query.resp.bits.rep_frm_fetch && 1074 RegNext(io.csrCtrl.ldld_vio_check_enable) 1075 1076 val s3_rep_info = WireInit(s3_in.rep_info) 1077 s3_rep_info.wpu_fail := s3_in.rep_info.wpu_fail && !s3_fwd_frm_d_chan_valid && s3_troublem 1078 s3_rep_info.bank_conflict := s3_in.rep_info.bank_conflict && !s3_fwd_frm_d_chan_valid && s3_troublem 1079 s3_rep_info.dcache_miss := s3_in.rep_info.dcache_miss && !s3_fwd_frm_d_chan_valid && s3_troublem 1080 val s3_rep_frm_fetch = s3_vp_match_fail || s3_ldld_rep_inst 1081 val s3_sel_rep_cause = PriorityEncoderOH(s3_rep_info.cause.asUInt) 1082 val s3_force_rep = s3_sel_rep_cause(LoadReplayCauses.C_TM) && 1083 !s3_in.uop.exceptionVec(loadAddrMisaligned) && 1084 s3_troublem 1085 1086 val s3_ld_exception = ExceptionNO.selectByFu(s3_in.uop.exceptionVec, LduCfg).asUInt.orR && s3_ld_flow 1087 val s3_st_exception = ExceptionNO.selectByFu(s3_in.uop.exceptionVec, StaCfg).asUInt.orR && !s3_ld_flow 1088 val s3_exception = s3_ld_exception || s3_st_exception 1089 when ((s3_ld_exception || s3_dly_ld_err || s3_rep_frm_fetch) && !s3_force_rep) { 1090 io.ldu_io.lsq.ldin.bits.rep_info.cause := 0.U.asTypeOf(s3_rep_info.cause.cloneType) 1091 } .otherwise { 1092 io.ldu_io.lsq.ldin.bits.rep_info.cause := VecInit(s3_sel_rep_cause.asBools) 1093 } 1094 1095 // Int flow, if hit, will be writebacked at s3 1096 s3_out.valid := s3_valid && 1097 (!s3_ld_flow || !io.ldu_io.lsq.ldin.bits.rep_info.need_rep && !s3_in.mmio) 1098 s3_out.bits.uop := s3_in.uop 1099 s3_out.bits.uop.exceptionVec(loadAccessFault) := s3_dly_ld_err || s3_in.uop.exceptionVec(loadAccessFault) 1100 s3_out.bits.uop.replayInst := s3_rep_frm_fetch 1101 s3_out.bits.data := s3_in.data 1102 s3_out.bits.debug.isMMIO := s3_in.mmio 1103 s3_out.bits.debug.isPerfCnt := false.B 1104 s3_out.bits.debug.paddr := s3_in.paddr 1105 s3_out.bits.debug.vaddr := s3_in.vaddr 1106 1107 when (s3_force_rep) { 1108 s3_out.bits.uop.exceptionVec := 0.U.asTypeOf(s3_in.uop.exceptionVec.cloneType) 1109 } 1110 1111 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1112 io.ldu_io.lsq.ldin.bits.uop := s3_out.bits.uop 1113 1114 val s3_revoke = s3_exception || io.ldu_io.lsq.ldin.bits.rep_info.need_rep 1115 io.ldu_io.lsq.ldld_nuke_query.revoke := s3_revoke 1116 io.ldu_io.lsq.stld_nuke_query.revoke := s3_revoke 1117 1118 // feedback slow 1119 s3_fast_rep := RegNext(s2_fast_rep) && 1120 !s3_in.feedbacked && 1121 !s3_in.lateKill && 1122 !s3_rep_frm_fetch && 1123 !s3_exception 1124 1125 val s3_fb_no_waiting = !s3_in.isLoadReplay && !(s3_fast_rep && io.ldu_io.fast_rep_out.ready) && !s3_in.feedbacked 1126 1127 // 1128 io.feedback_slow.valid := s3_valid && !s3_in.uop.robIdx.needFlush(io.redirect) && s3_fb_no_waiting && s3_ld_flow 1129 io.feedback_slow.bits.hit := !io.ldu_io.lsq.ldin.bits.rep_info.need_rep || io.ldu_io.lsq.ldin.ready 1130 io.feedback_slow.bits.flushState := s3_in.ptwBack 1131 io.feedback_slow.bits.robIdx := s3_in.uop.robIdx 1132 io.feedback_slow.bits.sourceType := RSFeedbackType.lrqFull 1133 io.feedback_slow.bits.dataInvalidSqIdx := DontCare 1134 1135 io.ldu_io.ldCancel.ld2Cancel.valid := s3_valid && ( 1136 (io.ldu_io.lsq.ldin.bits.rep_info.need_rep && s3_in.isFirstIssue) || 1137 s3_in.mmio 1138 ) 1139 io.ldu_io.ldCancel.ld2Cancel.bits := s3_in.deqPortIdx 1140 1141 // data from dcache hit 1142 val s3_ld_raw_data_frm_cache = Wire(new LoadDataFromDcacheBundle) 1143 s3_ld_raw_data_frm_cache.respDcacheData := io.dcache.resp.bits.data_delayed 1144 s3_ld_raw_data_frm_cache.forwardMask := RegEnable(s2_fwd_mask, s2_valid) 1145 s3_ld_raw_data_frm_cache.forwardData := RegEnable(s2_fwd_data, s2_valid) 1146 s3_ld_raw_data_frm_cache.uop := RegEnable(s2_out.uop, s2_valid) 1147 s3_ld_raw_data_frm_cache.addrOffset := RegEnable(s2_out.paddr(3, 0), s2_valid) 1148 s3_ld_raw_data_frm_cache.forward_D := RegEnable(s2_fwd_frm_d_chan, false.B, s2_valid) || s3_fwd_frm_d_chan_valid 1149 s3_ld_raw_data_frm_cache.forwardData_D := Mux(s3_fwd_frm_d_chan_valid, s3_fwd_data_frm_d_chan, RegEnable(s2_fwd_data_frm_d_chan, s2_valid)) 1150 s3_ld_raw_data_frm_cache.forward_mshr := RegEnable(s2_fwd_frm_mshr, false.B, s2_valid) 1151 s3_ld_raw_data_frm_cache.forwardData_mshr := RegEnable(s2_fwd_data_frm_mshr, s2_valid) 1152 s3_ld_raw_data_frm_cache.forward_result_valid := RegEnable(s2_fwd_data_valid, false.B, s2_valid) 1153 1154 val s3_merged_data_frm_cache = s3_ld_raw_data_frm_cache.mergedData() 1155 val s3_picked_data_frm_cache = LookupTree(s3_ld_raw_data_frm_cache.addrOffset, List( 1156 "b0000".U -> s3_merged_data_frm_cache(63, 0), 1157 "b0001".U -> s3_merged_data_frm_cache(63, 8), 1158 "b0010".U -> s3_merged_data_frm_cache(63, 16), 1159 "b0011".U -> s3_merged_data_frm_cache(63, 24), 1160 "b0100".U -> s3_merged_data_frm_cache(63, 32), 1161 "b0101".U -> s3_merged_data_frm_cache(63, 40), 1162 "b0110".U -> s3_merged_data_frm_cache(63, 48), 1163 "b0111".U -> s3_merged_data_frm_cache(63, 56), 1164 "b1000".U -> s3_merged_data_frm_cache(127, 64), 1165 "b1001".U -> s3_merged_data_frm_cache(127, 72), 1166 "b1010".U -> s3_merged_data_frm_cache(127, 80), 1167 "b1011".U -> s3_merged_data_frm_cache(127, 88), 1168 "b1100".U -> s3_merged_data_frm_cache(127, 96), 1169 "b1101".U -> s3_merged_data_frm_cache(127, 104), 1170 "b1110".U -> s3_merged_data_frm_cache(127, 112), 1171 "b1111".U -> s3_merged_data_frm_cache(127, 120) 1172 )) 1173 val s3_ld_data_frm_cache = rdataHelper(s3_ld_raw_data_frm_cache.uop, s3_picked_data_frm_cache) 1174 1175 // FIXME: add 1 cycle delay ? 1176 io.out_to_iq.bits := s3_out.bits 1177 io.out_to_iq.bits.data := s3_ld_data_frm_cache 1178 io.out_to_iq.valid := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) 1179 1180 // fast load to load forward 1181 io.ldu_io.l2l_fwd_out.valid := s3_out.valid && !s3_in.lateKill && s3_ld_flow 1182 io.ldu_io.l2l_fwd_out.data := s3_ld_data_frm_cache 1183 io.ldu_io.l2l_fwd_out.dly_ld_err := s3_dly_ld_err // ecc delayed error 1184 1185 // hybrid unit writeback to rob 1186 // delay params 1187 val SelectGroupSize = RollbackGroupSize 1188 val lgSelectGroupSize = log2Ceil(SelectGroupSize) 1189 val TotalSelectCycles = scala.math.ceil(log2Ceil(LoadQueueRAWSize).toFloat / lgSelectGroupSize).toInt + 1 1190 val TotalDelayCycles = TotalSelectCycles - 2 1191 1192 // writeback 1193 val sx_valid = Wire(Vec(TotalDelayCycles + 1, Bool())) 1194 val sx_ready = Wire(Vec(TotalDelayCycles + 1, Bool())) 1195 val sx_in = Wire(Vec(TotalDelayCycles + 1, new MemExuOutput)) 1196 1197 sx_can_go := sx_ready.head 1198 for (i <- 0 until TotalDelayCycles + 1) { 1199 if (i == 0) { 1200 sx_valid(i) := s3_valid 1201 sx_in(i) := s3_out.bits 1202 sx_ready(i) := !s3_valid(i) || sx_in(i).uop.robIdx.needFlush(io.redirect) || (if (TotalDelayCycles == 0) io.out_to_rob.ready else sx_ready(i+1)) 1203 } else { 1204 val cur_kill = sx_in(i).uop.robIdx.needFlush(io.redirect) 1205 val cur_can_go = (if (i == TotalDelayCycles) io.out_to_rob.ready else sx_ready(i+1)) 1206 val cur_fire = sx_valid(i) && !cur_kill && cur_can_go 1207 val prev_fire = sx_valid(i-1) && !sx_in(i-1).uop.robIdx.needFlush(io.redirect) && sx_ready(i) 1208 1209 sx_ready(i) := !sx_valid(i) || cur_kill || (if (i == TotalDelayCycles) io.out_to_rob.ready else sx_ready(i+1)) 1210 val sx_valid_can_go = prev_fire || cur_fire || cur_kill 1211 sx_valid(i) := RegEnable(Mux(prev_fire, true.B, false.B), sx_valid_can_go) 1212 sx_in(i) := RegEnable(sx_in(i-1), prev_fire) 1213 } 1214 } 1215 1216 val sx_last_valid = sx_valid.takeRight(1).head 1217 val sx_last_ready = sx_ready.takeRight(1).head 1218 val sx_last_in = sx_in.takeRight(1).head 1219 1220 sx_last_ready := !sx_last_valid || sx_last_in.uop.robIdx.needFlush(io.redirect) || io.out_to_rob.ready 1221 io.out_to_rob.valid := sx_last_valid && !sx_last_in.uop.robIdx.needFlush(io.redirect) 1222 io.out_to_rob.bits := sx_last_in 1223 1224 // trigger 1225 val ld_trigger = FuType.isLoad(io.out_to_rob.bits.uop.fuType) 1226 val last_valid_data = RegEnable(io.out_to_rob.bits.data, io.out_to_rob.fire) 1227 val hit_ld_addr_trig_hit_vec = Wire(Vec(3, Bool())) 1228 val lq_ld_addr_trig_hit_vec = RegNext(io.ldu_io.lsq.trigger.lqLoadAddrTriggerHitVec) 1229 (0 until 3).map{i => { 1230 val tdata2 = RegNext(RegNext(io.ldu_io.trigger(i).tdata2)) 1231 val matchType = RegNext(RegNext(io.ldu_io.trigger(i).matchType)) 1232 val tEnable = RegNext(RegNext(io.ldu_io.trigger(i).tEnable)) 1233 1234 hit_ld_addr_trig_hit_vec(i) := TriggerCmp(RegNext(s3_in.vaddr), tdata2, matchType, tEnable) 1235 io.ldu_io.trigger(i).addrHit := Mux(io.out_to_rob.valid && ld_trigger, hit_ld_addr_trig_hit_vec(i), lq_ld_addr_trig_hit_vec(i)) 1236 io.ldu_io.trigger(i).lastDataHit := TriggerCmp(last_valid_data, tdata2, matchType, tEnable) 1237 }} 1238 io.ldu_io.lsq.trigger.hitLoadAddrTriggerHitVec := hit_ld_addr_trig_hit_vec 1239 1240 // FIXME: please move this part to LoadQueueReplay 1241 io.ldu_io.debug_ls := DontCare 1242 1243 // Topdown 1244 io.ldu_io.lsTopdownInfo.s1.robIdx := s1_in.uop.robIdx.value 1245 io.ldu_io.lsTopdownInfo.s1.vaddr_valid := s1_valid && s1_in.hasROBEntry 1246 io.ldu_io.lsTopdownInfo.s1.vaddr_bits := s1_vaddr 1247 io.ldu_io.lsTopdownInfo.s2.robIdx := s2_in.uop.robIdx.value 1248 io.ldu_io.lsTopdownInfo.s2.paddr_valid := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss 1249 io.ldu_io.lsTopdownInfo.s2.paddr_bits := s2_in.paddr 1250 io.ldu_io.lsTopdownInfo.s2.first_real_miss := io.dcache.resp.bits.real_miss 1251 io.ldu_io.lsTopdownInfo.s2.cache_miss_en := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated 1252 1253 // perf cnt 1254 XSPerfAccumulate("s0_in_valid", io.lsin.valid) 1255 XSPerfAccumulate("s0_in_block", io.lsin.valid && !io.lsin.fire) 1256 XSPerfAccumulate("s0_in_fire_first_issue", s0_valid && s0_isFirstIssue) 1257 XSPerfAccumulate("s0_lsq_fire_first_issue", io.ldu_io.replay.fire) 1258 XSPerfAccumulate("s0_ldu_fire_first_issue", io.lsin.fire && s0_isFirstIssue) 1259 XSPerfAccumulate("s0_fast_replay_issue", io.ldu_io.fast_rep_in.fire) 1260 XSPerfAccumulate("s0_stall_out", s0_valid && !s0_can_go) 1261 XSPerfAccumulate("s0_stall_dcache", s0_valid && !io.dcache.req.ready) 1262 XSPerfAccumulate("s0_addr_spec_success", s0_fire && s0_vaddr(VAddrBits-1, 12) === io.lsin.bits.src(0)(VAddrBits-1, 12)) 1263 XSPerfAccumulate("s0_addr_spec_failed", s0_fire && s0_vaddr(VAddrBits-1, 12) =/= io.lsin.bits.src(0)(VAddrBits-1, 12)) 1264 XSPerfAccumulate("s0_addr_spec_success_once", s0_fire && s0_vaddr(VAddrBits-1, 12) === io.lsin.bits.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 1265 XSPerfAccumulate("s0_addr_spec_failed_once", s0_fire && s0_vaddr(VAddrBits-1, 12) =/= io.lsin.bits.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 1266 XSPerfAccumulate("s0_forward_tl_d_channel", s0_out.forward_tlDchannel) 1267 XSPerfAccumulate("s0_hardware_prefetch_fire", s0_fire && s0_hw_prf_select) 1268 XSPerfAccumulate("s0_software_prefetch_fire", s0_fire && s0_prf && s0_int_iss_select) 1269 XSPerfAccumulate("s0_hardware_prefetch_blocked", io.ldu_io.prefetch_req.valid && !s0_hw_prf_select) 1270 XSPerfAccumulate("s0_hardware_prefetch_total", io.ldu_io.prefetch_req.valid) 1271 1272 XSPerfAccumulate("s1_in_valid", s1_valid) 1273 XSPerfAccumulate("s1_in_fire", s1_fire) 1274 XSPerfAccumulate("s1_in_fire_first_issue", s1_fire && s1_in.isFirstIssue) 1275 XSPerfAccumulate("s1_tlb_miss", s1_fire && s1_tlb_miss) 1276 XSPerfAccumulate("s1_tlb_miss_first_issue", s1_fire && s1_tlb_miss && s1_in.isFirstIssue) 1277 XSPerfAccumulate("s1_stall_out", s1_valid && !s1_can_go) 1278 XSPerfAccumulate("s1_late_kill", s1_valid && s1_fast_rep_kill) 1279 1280 XSPerfAccumulate("s2_in_valid", s2_valid) 1281 XSPerfAccumulate("s2_in_fire", s2_fire) 1282 XSPerfAccumulate("s2_in_fire_first_issue", s2_fire && s2_in.isFirstIssue) 1283 XSPerfAccumulate("s2_dcache_miss", s2_fire && io.dcache.resp.bits.miss) 1284 XSPerfAccumulate("s2_dcache_miss_first_issue", s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1285 XSPerfAccumulate("s2_dcache_real_miss_first_issue", s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1286 XSPerfAccumulate("s2_full_forward", s2_fire && s2_full_fwd) 1287 XSPerfAccumulate("s2_dcache_miss_full_forward", s2_fire && s2_dcache_miss) 1288 XSPerfAccumulate("s2_fwd_frm_d_can", s2_valid && s2_fwd_frm_d_chan) 1289 XSPerfAccumulate("s2_fwd_frm_d_chan_or_mshr", s2_valid && s2_fwd_frm_d_chan_or_mshr) 1290 XSPerfAccumulate("s2_stall_out", s2_fire && !s2_can_go) 1291 XSPerfAccumulate("s2_prefetch", s2_fire && s2_prf) 1292 XSPerfAccumulate("s2_prefetch_ignored", s2_fire && s2_prf && s2_mq_nack) // ignore prefetch for mshr full / miss req port conflict 1293 XSPerfAccumulate("s2_prefetch_miss", s2_fire && s2_prf && io.dcache.resp.bits.miss) // prefetch req miss in l1 1294 XSPerfAccumulate("s2_prefetch_hit", s2_fire && s2_prf && !io.dcache.resp.bits.miss) // prefetch req hit in l1 1295 XSPerfAccumulate("s2_prefetch_accept", s2_fire && s2_prf && io.dcache.resp.bits.miss && !s2_mq_nack) // prefetch a missed line in l1, and l1 accepted it 1296 XSPerfAccumulate("s2_forward_req", s2_fire && s2_in.forward_tlDchannel) 1297 XSPerfAccumulate("s2_successfully_forward_channel_D", s2_fire && s2_fwd_frm_d_chan && s2_fwd_data_valid) 1298 XSPerfAccumulate("s2_successfully_forward_mshr", s2_fire && s2_fwd_frm_mshr && s2_fwd_data_valid) 1299 1300 XSPerfAccumulate("s3_fwd_frm_d_chan", s3_valid && s3_fwd_frm_d_chan_valid) 1301 1302 XSPerfAccumulate("load_to_load_forward", s1_try_ptr_chasing && !s1_ptr_chasing_canceled) 1303 XSPerfAccumulate("load_to_load_forward_try", s1_try_ptr_chasing) 1304 XSPerfAccumulate("load_to_load_forward_fail", s1_cancel_ptr_chasing) 1305 XSPerfAccumulate("load_to_load_forward_fail_cancelled", s1_cancel_ptr_chasing && s1_ptr_chasing_canceled) 1306 XSPerfAccumulate("load_to_load_forward_fail_wakeup_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && s1_not_fast_match) 1307 XSPerfAccumulate("load_to_load_forward_fail_op_not_ld", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && s1_fu_op_type_not_ld) 1308 XSPerfAccumulate("load_to_load_forward_fail_addr_align", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && s1_addr_misaligned) 1309 XSPerfAccumulate("load_to_load_forward_fail_set_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && !s1_addr_misaligned && s1_addr_mismatch) 1310 1311 // bug lyq: some signals in perfEvents are no longer suitable for the current MemBlock design 1312 // hardware performance counter 1313 val perfEvents = Seq( 1314 ("load_s0_in_fire ", s0_fire ), 1315 ("load_to_load_forward ", s1_fire && s1_try_ptr_chasing && !s1_ptr_chasing_canceled ), 1316 ("stall_dcache ", s0_valid && s0_can_go && !io.dcache.req.ready ), 1317 ("load_s1_in_fire ", s0_fire ), 1318 ("load_s1_tlb_miss ", s1_fire && io.tlb.resp.bits.miss ), 1319 ("load_s2_in_fire ", s1_fire ), 1320 ("load_s2_dcache_miss ", s2_fire && io.dcache.resp.bits.miss ), 1321 ) 1322 generatePerfEvent() 1323 1324 when(io.out_to_iq.fire){ 1325 XSDebug("out_to_iq %x\n", io.out_to_iq.bits.uop.pc) 1326 } 1327 // end 1328}