1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.Bundles.{DynInst, MemExuInput, MemExuOutput} 27import xiangshan.backend.fu.PMPRespBundle 28import xiangshan.backend.fu.FuConfig._ 29import xiangshan.backend.fu.FuType 30import xiangshan.backend.ctrlblock.{DebugLsInfoBundle, LsTopdownInfo} 31import xiangshan.backend.rob.RobPtr 32import xiangshan.backend.ctrlblock.DebugLsInfoBundle 33import xiangshan.backend.fu.NewCSR._ 34import xiangshan.backend.fu.util.SdtrigExt 35import xiangshan.cache._ 36import xiangshan.cache.wpu.ReplayCarry 37import xiangshan.cache.mmu._ 38import xiangshan.mem.mdp._ 39 40class LoadToLsqReplayIO(implicit p: Parameters) extends XSBundle 41 with HasDCacheParameters 42 with HasTlbConst 43{ 44 // mshr refill index 45 val mshr_id = UInt(log2Up(cfg.nMissEntries).W) 46 // get full data from store queue and sbuffer 47 val full_fwd = Bool() 48 // wait for data from store inst's store queue index 49 val data_inv_sq_idx = new SqPtr 50 // wait for address from store queue index 51 val addr_inv_sq_idx = new SqPtr 52 // replay carry 53 val rep_carry = new ReplayCarry(nWays) 54 // data in last beat 55 val last_beat = Bool() 56 // replay cause 57 val cause = Vec(LoadReplayCauses.allCauses, Bool()) 58 // performance debug information 59 val debug = new PerfDebugInfo 60 // tlb hint 61 val tlb_id = UInt(log2Up(loadfiltersize).W) 62 val tlb_full = Bool() 63 64 // alias 65 def mem_amb = cause(LoadReplayCauses.C_MA) 66 def tlb_miss = cause(LoadReplayCauses.C_TM) 67 def fwd_fail = cause(LoadReplayCauses.C_FF) 68 def dcache_rep = cause(LoadReplayCauses.C_DR) 69 def dcache_miss = cause(LoadReplayCauses.C_DM) 70 def wpu_fail = cause(LoadReplayCauses.C_WF) 71 def bank_conflict = cause(LoadReplayCauses.C_BC) 72 def rar_nack = cause(LoadReplayCauses.C_RAR) 73 def raw_nack = cause(LoadReplayCauses.C_RAW) 74 def misalign_nack = cause(LoadReplayCauses.C_MF) 75 def nuke = cause(LoadReplayCauses.C_NK) 76 def need_rep = cause.asUInt.orR 77} 78 79 80class LoadToLsqIO(implicit p: Parameters) extends XSBundle { 81 // ldu -> lsq UncacheBuffer 82 val ldin = DecoupledIO(new LqWriteBundle) 83 // uncache-mmio -> ldu 84 val uncache = Flipped(DecoupledIO(new MemExuOutput)) 85 val ld_raw_data = Input(new LoadDataFromLQBundle) 86 // uncache-nc -> ldu 87 val nc_ldin = Flipped(DecoupledIO(new LsPipelineBundle)) 88 // storequeue -> ldu 89 val forward = new PipeLoadForwardQueryIO 90 // ldu -> lsq LQRAW 91 val stld_nuke_query = new LoadNukeQueryIO 92 // ldu -> lsq LQRAR 93 val ldld_nuke_query = new LoadNukeQueryIO 94} 95 96class LoadToLoadIO(implicit p: Parameters) extends XSBundle { 97 val valid = Bool() 98 val data = UInt(XLEN.W) // load to load fast path is limited to ld (64 bit) used as vaddr src1 only 99 val dly_ld_err = Bool() 100} 101 102class LoadUnitTriggerIO(implicit p: Parameters) extends XSBundle { 103 val tdata2 = Input(UInt(64.W)) 104 val matchType = Input(UInt(2.W)) 105 val tEnable = Input(Bool()) // timing is calculated before this 106 val addrHit = Output(Bool()) 107} 108 109class LoadUnit(implicit p: Parameters) extends XSModule 110 with HasLoadHelper 111 with HasPerfEvents 112 with HasDCacheParameters 113 with HasCircularQueuePtrHelper 114 with HasVLSUParameters 115 with SdtrigExt 116{ 117 val io = IO(new Bundle() { 118 // control 119 val redirect = Flipped(ValidIO(new Redirect)) 120 val csrCtrl = Flipped(new CustomCSRCtrlIO) 121 122 // int issue path 123 val ldin = Flipped(Decoupled(new MemExuInput)) 124 val ldout = Decoupled(new MemExuOutput) 125 126 // vec issue path 127 val vecldin = Flipped(Decoupled(new VecPipeBundle)) 128 val vecldout = Decoupled(new VecPipelineFeedbackIO(isVStore = false)) 129 130 // misalignBuffer issue path 131 val misalign_ldin = Flipped(Decoupled(new LsPipelineBundle)) 132 val misalign_ldout = Valid(new LqWriteBundle) 133 134 // data path 135 val tlb = new TlbRequestIO(2) 136 val pmp = Flipped(new PMPRespBundle()) // arrive same to tlb now 137 val dcache = new DCacheLoadIO 138 val sbuffer = new LoadForwardQueryIO 139 val ubuffer = new LoadForwardQueryIO 140 val lsq = new LoadToLsqIO 141 val tl_d_channel = Input(new DcacheToLduForwardIO) 142 val forward_mshr = Flipped(new LduToMissqueueForwardIO) 143 // val refill = Flipped(ValidIO(new Refill)) 144 val l2_hint = Input(Valid(new L2ToL1Hint)) 145 val tlb_hint = Flipped(new TlbHintReq) 146 // fast wakeup 147 // TODO: implement vector fast wakeup 148 val fast_uop = ValidIO(new DynInst) // early wakeup signal generated in load_s1, send to RS in load_s2 149 150 // trigger 151 val fromCsrTrigger = Input(new CsrTriggerBundle) 152 153 // prefetch 154 val prefetch_train = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to sms 155 val prefetch_train_l1 = ValidIO(new LdPrefetchTrainBundle()) // provide prefetch info to stream & stride 156 // speculative for gated control 157 val s1_prefetch_spec = Output(Bool()) 158 val s2_prefetch_spec = Output(Bool()) 159 160 val prefetch_req = Flipped(ValidIO(new L1PrefetchReq)) // hardware prefetch to l1 cache req 161 val canAcceptLowConfPrefetch = Output(Bool()) 162 val canAcceptHighConfPrefetch = Output(Bool()) 163 164 // ifetchPrefetch 165 val ifetchPrefetch = ValidIO(new SoftIfetchPrefetchBundle) 166 167 // load to load fast path 168 val l2l_fwd_in = Input(new LoadToLoadIO) 169 val l2l_fwd_out = Output(new LoadToLoadIO) 170 171 val ld_fast_match = Input(Bool()) 172 val ld_fast_fuOpType = Input(UInt()) 173 val ld_fast_imm = Input(UInt(12.W)) 174 175 // rs feedback 176 val wakeup = ValidIO(new DynInst) 177 val feedback_fast = ValidIO(new RSFeedback) // stage 2 178 val feedback_slow = ValidIO(new RSFeedback) // stage 3 179 val ldCancel = Output(new LoadCancelIO()) // use to cancel the uops waked by this load, and cancel load 180 181 // load ecc error 182 val s3_dly_ld_err = Output(Bool()) // Note that io.s3_dly_ld_err and io.lsq.s3_dly_ld_err is different 183 184 // schedule error query 185 val stld_nuke_query = Flipped(Vec(StorePipelineWidth, Valid(new StoreNukeQueryIO))) 186 187 // queue-based replay 188 val replay = Flipped(Decoupled(new LsPipelineBundle)) 189 val lq_rep_full = Input(Bool()) 190 191 // misc 192 val s2_ptr_chasing = Output(Bool()) // provide right pc for hw prefetch 193 194 // Load fast replay path 195 val fast_rep_in = Flipped(Decoupled(new LqWriteBundle)) 196 val fast_rep_out = Decoupled(new LqWriteBundle) 197 198 // to misalign buffer 199 val misalign_buf = Decoupled(new LqWriteBundle) 200 201 // Load RAR rollback 202 val rollback = Valid(new Redirect) 203 204 // perf 205 val debug_ls = Output(new DebugLsInfoBundle) 206 val lsTopdownInfo = Output(new LsTopdownInfo) 207 val correctMissTrain = Input(Bool()) 208 }) 209 210 val s1_ready, s2_ready, s3_ready = WireInit(false.B) 211 212 // Pipeline 213 // -------------------------------------------------------------------------------- 214 // stage 0 215 // -------------------------------------------------------------------------------- 216 // generate addr, use addr to query DCache and DTLB 217 val s0_valid = Wire(Bool()) 218 val s0_mmio_select = Wire(Bool()) 219 val s0_nc_select = Wire(Bool()) 220 val s0_misalign_select= Wire(Bool()) 221 val s0_kill = Wire(Bool()) 222 val s0_can_go = s1_ready 223 val s0_fire = s0_valid && s0_can_go 224 val s0_mmio_fire = s0_mmio_select && s0_can_go 225 val s0_nc_fire = s0_nc_select && s0_can_go 226 val s0_out = Wire(new LqWriteBundle) 227 val s0_tlb_valid = Wire(Bool()) 228 val s0_tlb_hlv = Wire(Bool()) 229 val s0_tlb_hlvx = Wire(Bool()) 230 val s0_tlb_vaddr = Wire(UInt(VAddrBits.W)) 231 val s0_tlb_fullva = Wire(UInt(XLEN.W)) 232 val s0_dcache_vaddr = Wire(UInt(VAddrBits.W)) 233 val s0_is128bit = Wire(Bool()) 234 val s0_misalign_wakeup_fire = s0_misalign_select && s0_can_go && io.misalign_ldin.bits.misalignNeedWakeUp 235 236 // flow source bundle 237 class FlowSource extends Bundle { 238 val vaddr = UInt(VAddrBits.W) 239 val mask = UInt((VLEN/8).W) 240 val uop = new DynInst 241 val try_l2l = Bool() 242 val has_rob_entry = Bool() 243 val rep_carry = new ReplayCarry(nWays) 244 val mshrid = UInt(log2Up(cfg.nMissEntries).W) 245 val isFirstIssue = Bool() 246 val fast_rep = Bool() 247 val ld_rep = Bool() 248 val l2l_fwd = Bool() 249 val prf = Bool() 250 val prf_rd = Bool() 251 val prf_wr = Bool() 252 val prf_i = Bool() 253 val sched_idx = UInt(log2Up(LoadQueueReplaySize+1).W) 254 // Record the issue port idx of load issue queue. This signal is used by load cancel. 255 val deqPortIdx = UInt(log2Ceil(LoadPipelineWidth).W) 256 val frm_mabuf = Bool() 257 // vec only 258 val isvec = Bool() 259 val is128bit = Bool() 260 val uop_unit_stride_fof = Bool() 261 val reg_offset = UInt(vOffsetBits.W) 262 val vecActive = Bool() // 1: vector active element or scala mem operation, 0: vector not active element 263 val is_first_ele = Bool() 264 // val flowPtr = new VlflowPtr 265 val usSecondInv = Bool() 266 val mbIndex = UInt(vlmBindexBits.W) 267 val elemIdx = UInt(elemIdxBits.W) 268 val elemIdxInsideVd = UInt(elemIdxBits.W) 269 val alignedType = UInt(alignTypeBits.W) 270 val vecBaseVaddr = UInt(VAddrBits.W) 271 //for Svpbmt NC 272 val isnc = Bool() 273 val paddr = UInt(PAddrBits.W) 274 val data = UInt((VLEN+1).W) 275 } 276 val s0_sel_src = Wire(new FlowSource) 277 278 // load flow select/gen 279 // src 0: misalignBuffer load (io.misalign_ldin) 280 // src 1: super load replayed by LSQ (cache miss replay) (io.replay) 281 // src 2: fast load replay (io.fast_rep_in) 282 // src 3: mmio (io.lsq.uncache) 283 // src 4: nc (io.lsq.nc_ldin) 284 // src 5: load replayed by LSQ (io.replay) 285 // src 6: hardware prefetch from prefetchor (high confidence) (io.prefetch) 286 // NOTE: Now vec/int loads are sent from same RS 287 // A vec load will be splited into multiple uops, 288 // so as long as one uop is issued, 289 // the other uops should have higher priority 290 // src 7: vec read from RS (io.vecldin) 291 // src 8: int read / software prefetch first issue from RS (io.in) 292 // src 9: load try pointchaising when no issued or replayed load (io.fastpath) 293 // src10: hardware prefetch from prefetchor (high confidence) (io.prefetch) 294 // priority: high to low 295 val s0_rep_stall = io.ldin.valid && isAfter(io.replay.bits.uop.lqIdx, io.ldin.bits.uop.lqIdx) || 296 io.vecldin.valid && isAfter(io.replay.bits.uop.lqIdx, io.vecldin.bits.uop.lqIdx) 297 private val SRC_NUM = 11 298 private val Seq( 299 mab_idx, super_rep_idx, fast_rep_idx, mmio_idx, nc_idx, lsq_rep_idx, 300 high_pf_idx, vec_iss_idx, int_iss_idx, l2l_fwd_idx, low_pf_idx 301 ) = (0 until SRC_NUM).toSeq 302 // load flow source valid 303 val s0_src_valid_vec = WireInit(VecInit(Seq( 304 io.misalign_ldin.valid, 305 io.replay.valid && io.replay.bits.forward_tlDchannel, 306 io.fast_rep_in.valid, 307 io.lsq.uncache.valid, 308 io.lsq.nc_ldin.valid, 309 io.replay.valid && !io.replay.bits.forward_tlDchannel && !s0_rep_stall, 310 io.prefetch_req.valid && io.prefetch_req.bits.confidence > 0.U, 311 io.vecldin.valid, 312 io.ldin.valid, // int flow first issue or software prefetch 313 io.l2l_fwd_in.valid, 314 io.prefetch_req.valid && io.prefetch_req.bits.confidence === 0.U, 315 ))) 316 // load flow source ready 317 val s0_src_ready_vec = Wire(Vec(SRC_NUM, Bool())) 318 s0_src_ready_vec(0) := true.B 319 for(i <- 1 until SRC_NUM){ 320 s0_src_ready_vec(i) := !s0_src_valid_vec.take(i).reduce(_ || _) 321 } 322 // load flow source select (OH) 323 val s0_src_select_vec = WireInit(VecInit((0 until SRC_NUM).map{i => s0_src_valid_vec(i) && s0_src_ready_vec(i)})) 324 val s0_hw_prf_select = s0_src_select_vec(high_pf_idx) || s0_src_select_vec(low_pf_idx) 325 326 val s0_tlb_no_query = s0_hw_prf_select || s0_sel_src.prf_i || 327 s0_src_select_vec(fast_rep_idx) || s0_src_select_vec(mmio_idx) || 328 s0_src_select_vec(nc_idx) 329 s0_valid := !s0_kill && (s0_src_select_vec(nc_idx) || (( 330 s0_src_valid_vec(mab_idx) || 331 s0_src_valid_vec(super_rep_idx) || 332 s0_src_valid_vec(fast_rep_idx) || 333 s0_src_valid_vec(lsq_rep_idx) || 334 s0_src_valid_vec(high_pf_idx) || 335 s0_src_valid_vec(vec_iss_idx) || 336 s0_src_valid_vec(int_iss_idx) || 337 s0_src_valid_vec(l2l_fwd_idx) || 338 s0_src_valid_vec(low_pf_idx) 339 ) && !s0_src_select_vec(mmio_idx) && io.dcache.req.ready)) 340 341 s0_mmio_select := s0_src_select_vec(mmio_idx) && !s0_kill 342 s0_nc_select := s0_src_select_vec(nc_idx) && !s0_kill 343 //judgment: is NC with data or not. 344 //If true, it's from `io.lsq.nc_ldin` or `io.fast_rep_in` 345 val s0_nc_with_data = s0_sel_src.isnc && !s0_kill 346 s0_misalign_select := s0_src_select_vec(mab_idx) && !s0_kill 347 348 // if is hardware prefetch or fast replay, don't send valid to tlb 349 s0_tlb_valid := ( 350 s0_src_valid_vec(mab_idx) || 351 s0_src_valid_vec(super_rep_idx) || 352 s0_src_valid_vec(lsq_rep_idx) || 353 s0_src_valid_vec(vec_iss_idx) || 354 s0_src_valid_vec(int_iss_idx) || 355 s0_src_valid_vec(l2l_fwd_idx) 356 ) && io.dcache.req.ready 357 358 // which is S0's out is ready and dcache is ready 359 val s0_try_ptr_chasing = s0_src_select_vec(l2l_fwd_idx) 360 val s0_do_try_ptr_chasing = s0_try_ptr_chasing && s0_can_go && io.dcache.req.ready 361 val s0_ptr_chasing_vaddr = io.l2l_fwd_in.data(5, 0) +& io.ld_fast_imm(5, 0) 362 val s0_ptr_chasing_canceled = WireInit(false.B) 363 s0_kill := s0_ptr_chasing_canceled 364 365 // prefetch related ctrl signal 366 io.canAcceptLowConfPrefetch := s0_src_ready_vec(low_pf_idx) && io.dcache.req.ready 367 io.canAcceptHighConfPrefetch := s0_src_ready_vec(high_pf_idx) && io.dcache.req.ready 368 369 // query DTLB 370 io.tlb.req.valid := s0_tlb_valid 371 io.tlb.req.bits.cmd := Mux(s0_sel_src.prf, 372 Mux(s0_sel_src.prf_wr, TlbCmd.write, TlbCmd.read), 373 TlbCmd.read 374 ) 375 io.tlb.req.bits.isPrefetch := s0_sel_src.prf 376 io.tlb.req.bits.vaddr := s0_tlb_vaddr 377 io.tlb.req.bits.fullva := s0_tlb_fullva 378 io.tlb.req.bits.checkfullva := s0_src_select_vec(vec_iss_idx) || s0_src_select_vec(int_iss_idx) 379 io.tlb.req.bits.hyperinst := s0_tlb_hlv 380 io.tlb.req.bits.hlvx := s0_tlb_hlvx 381 io.tlb.req.bits.size := Mux(s0_sel_src.isvec, s0_sel_src.alignedType(2,0), LSUOpType.size(s0_sel_src.uop.fuOpType)) 382 io.tlb.req.bits.kill := s0_kill || s0_tlb_no_query // if does not need to be translated, kill it 383 io.tlb.req.bits.memidx.is_ld := true.B 384 io.tlb.req.bits.memidx.is_st := false.B 385 io.tlb.req.bits.memidx.idx := s0_sel_src.uop.lqIdx.value 386 io.tlb.req.bits.debug.robIdx := s0_sel_src.uop.robIdx 387 io.tlb.req.bits.no_translate := s0_tlb_no_query // hardware prefetch and fast replay does not need to be translated, need this signal for pmp check 388 io.tlb.req.bits.debug.pc := s0_sel_src.uop.pc 389 io.tlb.req.bits.debug.isFirstIssue := s0_sel_src.isFirstIssue 390 391 // query DCache 392 io.dcache.req.valid := s0_valid && !s0_sel_src.prf_i && !s0_nc_with_data 393 io.dcache.req.bits.cmd := Mux(s0_sel_src.prf_rd, 394 MemoryOpConstants.M_PFR, 395 Mux(s0_sel_src.prf_wr, MemoryOpConstants.M_PFW, MemoryOpConstants.M_XRD) 396 ) 397 io.dcache.req.bits.vaddr := s0_dcache_vaddr 398 io.dcache.req.bits.vaddr_dup := s0_dcache_vaddr 399 io.dcache.req.bits.mask := s0_sel_src.mask 400 io.dcache.req.bits.data := DontCare 401 io.dcache.req.bits.isFirstIssue := s0_sel_src.isFirstIssue 402 io.dcache.req.bits.instrtype := Mux(s0_sel_src.prf, DCACHE_PREFETCH_SOURCE.U, LOAD_SOURCE.U) 403 io.dcache.req.bits.debug_robIdx := s0_sel_src.uop.robIdx.value 404 io.dcache.req.bits.replayCarry := s0_sel_src.rep_carry 405 io.dcache.req.bits.id := DontCare // TODO: update cache meta 406 io.dcache.req.bits.lqIdx := s0_sel_src.uop.lqIdx 407 io.dcache.pf_source := Mux(s0_hw_prf_select, io.prefetch_req.bits.pf_source.value, L1_HW_PREFETCH_NULL) 408 io.dcache.is128Req := s0_is128bit 409 410 // load flow priority mux 411 def fromNullSource(): FlowSource = { 412 val out = WireInit(0.U.asTypeOf(new FlowSource)) 413 out 414 } 415 416 def fromMisAlignBufferSource(src: LsPipelineBundle): FlowSource = { 417 val out = WireInit(0.U.asTypeOf(new FlowSource)) 418 out.vaddr := src.vaddr 419 out.mask := src.mask 420 out.uop := src.uop 421 out.try_l2l := false.B 422 out.has_rob_entry := false.B 423 out.rep_carry := src.replayCarry 424 out.mshrid := src.mshrid 425 out.frm_mabuf := true.B 426 out.isFirstIssue := false.B 427 out.fast_rep := false.B 428 out.ld_rep := false.B 429 out.l2l_fwd := false.B 430 out.prf := false.B 431 out.prf_rd := false.B 432 out.prf_wr := false.B 433 out.sched_idx := src.schedIndex 434 out.isvec := src.isvec 435 out.is128bit := src.is128bit 436 out.vecActive := true.B 437 out 438 } 439 440 def fromFastReplaySource(src: LqWriteBundle): FlowSource = { 441 val out = WireInit(0.U.asTypeOf(new FlowSource)) 442 out.vaddr := src.vaddr 443 out.paddr := src.paddr 444 out.mask := src.mask 445 out.uop := src.uop 446 out.try_l2l := false.B 447 out.has_rob_entry := src.hasROBEntry 448 out.rep_carry := src.rep_info.rep_carry 449 out.mshrid := src.rep_info.mshr_id 450 out.frm_mabuf := src.isFrmMisAlignBuf 451 out.isFirstIssue := false.B 452 out.fast_rep := true.B 453 out.ld_rep := src.isLoadReplay 454 out.l2l_fwd := false.B 455 out.prf := LSUOpType.isPrefetch(src.uop.fuOpType) && !src.isvec 456 out.prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 457 out.prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 458 out.prf_i := false.B 459 out.sched_idx := src.schedIndex 460 out.isvec := src.isvec 461 out.is128bit := src.is128bit 462 out.uop_unit_stride_fof := src.uop_unit_stride_fof 463 out.reg_offset := src.reg_offset 464 out.vecActive := src.vecActive 465 out.is_first_ele := src.is_first_ele 466 out.usSecondInv := src.usSecondInv 467 out.mbIndex := src.mbIndex 468 out.elemIdx := src.elemIdx 469 out.elemIdxInsideVd := src.elemIdxInsideVd 470 out.alignedType := src.alignedType 471 out.isnc := src.nc 472 out.data := src.data 473 out 474 } 475 476 // TODO: implement vector mmio 477 def fromMmioSource(src: MemExuOutput) = { 478 val out = WireInit(0.U.asTypeOf(new FlowSource)) 479 out.mask := 0.U 480 out.uop := src.uop 481 out.try_l2l := false.B 482 out.has_rob_entry := false.B 483 out.rep_carry := 0.U.asTypeOf(out.rep_carry) 484 out.mshrid := 0.U 485 out.frm_mabuf := false.B 486 out.isFirstIssue := false.B 487 out.fast_rep := false.B 488 out.ld_rep := false.B 489 out.l2l_fwd := false.B 490 out.prf := false.B 491 out.prf_rd := false.B 492 out.prf_wr := false.B 493 out.prf_i := false.B 494 out.sched_idx := 0.U 495 out.vecActive := true.B 496 out 497 } 498 499 def fromNcSource(src: LsPipelineBundle): FlowSource = { 500 val out = WireInit(0.U.asTypeOf(new FlowSource)) 501 out.vaddr := src.vaddr 502 out.paddr := src.paddr 503 out.mask := genVWmask(src.vaddr, src.uop.fuOpType(1,0)) 504 out.uop := src.uop 505 out.has_rob_entry := true.B 506 out.sched_idx := src.schedIndex 507 out.isvec := src.isvec 508 out.is128bit := src.is128bit 509 out.vecActive := src.vecActive 510 out.isnc := true.B 511 out.data := src.data 512 out 513 } 514 515 def fromNormalReplaySource(src: LsPipelineBundle): FlowSource = { 516 val out = WireInit(0.U.asTypeOf(new FlowSource)) 517 out.mask := Mux(src.isvec, src.mask, genVWmask(src.vaddr, src.uop.fuOpType(1, 0))) 518 out.uop := src.uop 519 out.try_l2l := false.B 520 out.has_rob_entry := true.B 521 out.rep_carry := src.replayCarry 522 out.mshrid := src.mshrid 523 out.frm_mabuf := false.B 524 out.isFirstIssue := false.B 525 out.fast_rep := false.B 526 out.ld_rep := true.B 527 out.l2l_fwd := false.B 528 out.prf := LSUOpType.isPrefetch(src.uop.fuOpType) && !src.isvec 529 out.prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 530 out.prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 531 out.prf_i := false.B 532 out.sched_idx := src.schedIndex 533 out.isvec := src.isvec 534 out.is128bit := src.is128bit 535 out.uop_unit_stride_fof := src.uop_unit_stride_fof 536 out.reg_offset := src.reg_offset 537 out.vecActive := src.vecActive 538 out.is_first_ele := src.is_first_ele 539 out.usSecondInv := src.usSecondInv 540 out.mbIndex := src.mbIndex 541 out.elemIdx := src.elemIdx 542 out.elemIdxInsideVd := src.elemIdxInsideVd 543 out.alignedType := src.alignedType 544 out 545 } 546 547 // TODO: implement vector prefetch 548 def fromPrefetchSource(src: L1PrefetchReq): FlowSource = { 549 val out = WireInit(0.U.asTypeOf(new FlowSource)) 550 out.mask := 0.U 551 out.uop := DontCare 552 out.try_l2l := false.B 553 out.has_rob_entry := false.B 554 out.rep_carry := 0.U.asTypeOf(out.rep_carry) 555 out.mshrid := 0.U 556 out.frm_mabuf := false.B 557 out.isFirstIssue := false.B 558 out.fast_rep := false.B 559 out.ld_rep := false.B 560 out.l2l_fwd := false.B 561 out.prf := true.B 562 out.prf_rd := !src.is_store 563 out.prf_wr := src.is_store 564 out.prf_i := false.B 565 out.sched_idx := 0.U 566 out 567 } 568 569 def fromVecIssueSource(src: VecPipeBundle): FlowSource = { 570 val out = WireInit(0.U.asTypeOf(new FlowSource)) 571 out.mask := src.mask 572 out.uop := src.uop 573 out.try_l2l := false.B 574 out.has_rob_entry := true.B 575 // TODO: VLSU, implement replay carry 576 out.rep_carry := 0.U.asTypeOf(out.rep_carry) 577 out.mshrid := 0.U 578 out.frm_mabuf := false.B 579 // TODO: VLSU, implement first issue 580// out.isFirstIssue := src.isFirstIssue 581 out.fast_rep := false.B 582 out.ld_rep := false.B 583 out.l2l_fwd := false.B 584 out.prf := false.B 585 out.prf_rd := false.B 586 out.prf_wr := false.B 587 out.prf_i := false.B 588 out.sched_idx := 0.U 589 // Vector load interface 590 out.isvec := true.B 591 // vector loads only access a single element at a time, so 128-bit path is not used for now 592 out.is128bit := is128Bit(src.alignedType) 593 out.uop_unit_stride_fof := src.uop_unit_stride_fof 594 // out.rob_idx_valid := src.rob_idx_valid 595 // out.inner_idx := src.inner_idx 596 // out.rob_idx := src.rob_idx 597 out.reg_offset := src.reg_offset 598 // out.offset := src.offset 599 out.vecActive := src.vecActive 600 out.is_first_ele := src.is_first_ele 601 // out.flowPtr := src.flowPtr 602 out.usSecondInv := src.usSecondInv 603 out.mbIndex := src.mBIndex 604 out.elemIdx := src.elemIdx 605 out.elemIdxInsideVd := src.elemIdxInsideVd 606 out.vecBaseVaddr := src.basevaddr 607 out.alignedType := src.alignedType 608 out 609 } 610 611 def fromIntIssueSource(src: MemExuInput): FlowSource = { 612 val out = WireInit(0.U.asTypeOf(new FlowSource)) 613 val addr = io.ldin.bits.src(0) + SignExt(io.ldin.bits.uop.imm(11, 0), VAddrBits) 614 out.mask := genVWmask(addr, src.uop.fuOpType(1,0)) 615 out.uop := src.uop 616 out.try_l2l := false.B 617 out.has_rob_entry := true.B 618 out.rep_carry := 0.U.asTypeOf(out.rep_carry) 619 out.mshrid := 0.U 620 out.frm_mabuf := false.B 621 out.isFirstIssue := true.B 622 out.fast_rep := false.B 623 out.ld_rep := false.B 624 out.l2l_fwd := false.B 625 out.prf := LSUOpType.isPrefetch(src.uop.fuOpType) 626 out.prf_rd := src.uop.fuOpType === LSUOpType.prefetch_r 627 out.prf_wr := src.uop.fuOpType === LSUOpType.prefetch_w 628 out.prf_i := src.uop.fuOpType === LSUOpType.prefetch_i 629 out.sched_idx := 0.U 630 out.vecActive := true.B // true for scala load 631 out 632 } 633 634 // TODO: implement vector l2l 635 def fromLoadToLoadSource(src: LoadToLoadIO): FlowSource = { 636 val out = WireInit(0.U.asTypeOf(new FlowSource)) 637 out.mask := genVWmask(0.U, LSUOpType.ld) 638 // When there's no valid instruction from RS and LSQ, we try the load-to-load forwarding. 639 // Assume the pointer chasing is always ld. 640 out.uop.fuOpType := LSUOpType.ld 641 out.try_l2l := true.B 642 // we dont care out.isFirstIssue and out.rsIdx and s0_sqIdx in S0 when trying pointchasing 643 // because these signals will be updated in S1 644 out.has_rob_entry := false.B 645 out.mshrid := 0.U 646 out.frm_mabuf := false.B 647 out.rep_carry := 0.U.asTypeOf(out.rep_carry) 648 out.isFirstIssue := true.B 649 out.fast_rep := false.B 650 out.ld_rep := false.B 651 out.l2l_fwd := true.B 652 out.prf := false.B 653 out.prf_rd := false.B 654 out.prf_wr := false.B 655 out.prf_i := false.B 656 out.sched_idx := 0.U 657 out 658 } 659 660 // set default 661 val s0_src_selector = WireInit(s0_src_valid_vec) 662 if (!EnableLoadToLoadForward) { s0_src_selector(l2l_fwd_idx) := false.B } 663 val s0_src_format = Seq( 664 fromMisAlignBufferSource(io.misalign_ldin.bits), 665 fromNormalReplaySource(io.replay.bits), 666 fromFastReplaySource(io.fast_rep_in.bits), 667 fromMmioSource(io.lsq.uncache.bits), 668 fromNcSource(io.lsq.nc_ldin.bits), 669 fromNormalReplaySource(io.replay.bits), 670 fromPrefetchSource(io.prefetch_req.bits), 671 fromVecIssueSource(io.vecldin.bits), 672 fromIntIssueSource(io.ldin.bits), 673 (if (EnableLoadToLoadForward) fromLoadToLoadSource(io.l2l_fwd_in) else fromNullSource()), 674 fromPrefetchSource(io.prefetch_req.bits) 675 ) 676 s0_sel_src := ParallelPriorityMux(s0_src_selector, s0_src_format) 677 678 // fast replay and hardware prefetch don't need to query tlb 679 val int_issue_vaddr = io.ldin.bits.src(0) + SignExt(io.ldin.bits.uop.imm(11, 0), VAddrBits) 680 val int_vec_vaddr = Mux(s0_src_valid_vec(vec_iss_idx), io.vecldin.bits.vaddr(VAddrBits - 1, 0), int_issue_vaddr) 681 s0_tlb_vaddr := Mux( 682 s0_src_valid_vec(mab_idx), 683 io.misalign_ldin.bits.vaddr, 684 Mux( 685 s0_src_valid_vec(super_rep_idx) || s0_src_valid_vec(lsq_rep_idx), 686 io.replay.bits.vaddr, 687 int_vec_vaddr 688 ) 689 ) 690 s0_dcache_vaddr := Mux( 691 s0_src_select_vec(fast_rep_idx), io.fast_rep_in.bits.vaddr, 692 Mux(s0_hw_prf_select, io.prefetch_req.bits.getVaddr(), 693 Mux(s0_src_select_vec(nc_idx), io.lsq.nc_ldin.bits.vaddr, // not for dcache access, but for address alignment check 694 s0_tlb_vaddr)) 695 ) 696 697 val s0_alignType = Mux(s0_sel_src.isvec, s0_sel_src.alignedType(1,0), s0_sel_src.uop.fuOpType(1, 0)) 698 699 val s0_addr_aligned = LookupTree(s0_alignType, List( 700 "b00".U -> true.B, //b 701 "b01".U -> (s0_dcache_vaddr(0) === 0.U), //h 702 "b10".U -> (s0_dcache_vaddr(1, 0) === 0.U), //w 703 "b11".U -> (s0_dcache_vaddr(2, 0) === 0.U) //d 704 )) 705 // address align check 706 XSError(s0_sel_src.isvec && s0_dcache_vaddr(3, 0) =/= 0.U && s0_sel_src.alignedType(2), "unit-stride 128 bit element is not aligned!") 707 708 val s0_check_vaddr_low = s0_dcache_vaddr(4, 0) 709 val s0_check_vaddr_Up_low = LookupTree(s0_alignType, List( 710 "b00".U -> 0.U, 711 "b01".U -> 1.U, 712 "b10".U -> 3.U, 713 "b11".U -> 7.U 714 )) + s0_check_vaddr_low 715 //TODO vec? 716 val s0_rs_cross16Bytes = s0_check_vaddr_Up_low(4) =/= s0_check_vaddr_low(4) 717 val s0_misalignWith16Byte = !s0_rs_cross16Bytes && !s0_addr_aligned && !s0_hw_prf_select 718 val s0_misalignNeedWakeUp = s0_sel_src.frm_mabuf && io.misalign_ldin.bits.misalignNeedWakeUp 719 val s0_finalSplit = s0_sel_src.frm_mabuf && io.misalign_ldin.bits.isFinalSplit 720 s0_is128bit := s0_sel_src.is128bit || s0_misalignWith16Byte 721 722 // only first issue of int / vec load intructions need to check full vaddr 723 s0_tlb_fullva := Mux(s0_src_valid_vec(mab_idx), 724 io.misalign_ldin.bits.fullva, 725 Mux(s0_src_select_vec(vec_iss_idx), 726 io.vecldin.bits.vaddr, 727 Mux( 728 s0_src_select_vec(int_iss_idx), 729 io.ldin.bits.src(0) + SignExt(io.ldin.bits.uop.imm(11, 0), XLEN), 730 s0_dcache_vaddr 731 ) 732 ) 733 ) 734 735 s0_tlb_hlv := Mux( 736 s0_src_valid_vec(mab_idx), 737 LSUOpType.isHlv(io.misalign_ldin.bits.uop.fuOpType), 738 Mux( 739 s0_src_valid_vec(super_rep_idx) || s0_src_valid_vec(lsq_rep_idx), 740 LSUOpType.isHlv(io.replay.bits.uop.fuOpType), 741 Mux( 742 s0_src_valid_vec(int_iss_idx), 743 LSUOpType.isHlv(io.ldin.bits.uop.fuOpType), 744 false.B 745 ) 746 ) 747 ) 748 s0_tlb_hlvx := Mux( 749 s0_src_valid_vec(mab_idx), 750 LSUOpType.isHlvx(io.misalign_ldin.bits.uop.fuOpType), 751 Mux( 752 s0_src_valid_vec(super_rep_idx) || s0_src_valid_vec(lsq_rep_idx), 753 LSUOpType.isHlvx(io.replay.bits.uop.fuOpType), 754 Mux( 755 s0_src_valid_vec(int_iss_idx), 756 LSUOpType.isHlvx(io.ldin.bits.uop.fuOpType), 757 false.B 758 ) 759 ) 760 ) 761 762 // accept load flow if dcache ready (tlb is always ready) 763 // TODO: prefetch need writeback to loadQueueFlag 764 s0_out := DontCare 765 s0_out.vaddr := Mux(s0_nc_with_data, s0_sel_src.vaddr, s0_dcache_vaddr) 766 s0_out.fullva := s0_tlb_fullva 767 s0_out.mask := s0_sel_src.mask 768 s0_out.uop := s0_sel_src.uop 769 s0_out.isFirstIssue := s0_sel_src.isFirstIssue 770 s0_out.hasROBEntry := s0_sel_src.has_rob_entry 771 s0_out.isPrefetch := s0_sel_src.prf 772 s0_out.isHWPrefetch := s0_hw_prf_select 773 s0_out.isFastReplay := s0_sel_src.fast_rep 774 s0_out.isLoadReplay := s0_sel_src.ld_rep 775 s0_out.isFastPath := s0_sel_src.l2l_fwd 776 s0_out.mshrid := s0_sel_src.mshrid 777 s0_out.isvec := s0_sel_src.isvec 778 s0_out.is128bit := s0_is128bit 779 s0_out.isFrmMisAlignBuf := s0_sel_src.frm_mabuf 780 s0_out.uop_unit_stride_fof := s0_sel_src.uop_unit_stride_fof 781 s0_out.paddr := 782 Mux(s0_src_valid_vec(nc_idx), io.lsq.nc_ldin.bits.paddr, 783 Mux(s0_src_valid_vec(fast_rep_idx), io.fast_rep_in.bits.paddr, 784 Mux(s0_src_select_vec(int_iss_idx) && s0_sel_src.prf_i, 0.U, 785 io.prefetch_req.bits.paddr))) // only for nc, fast_rep, prefetch 786 s0_out.tlbNoQuery := s0_tlb_no_query 787 // s0_out.rob_idx_valid := s0_rob_idx_valid 788 // s0_out.inner_idx := s0_inner_idx 789 // s0_out.rob_idx := s0_rob_idx 790 s0_out.reg_offset := s0_sel_src.reg_offset 791 // s0_out.offset := s0_offset 792 s0_out.vecActive := s0_sel_src.vecActive 793 s0_out.usSecondInv := s0_sel_src.usSecondInv 794 s0_out.is_first_ele := s0_sel_src.is_first_ele 795 s0_out.elemIdx := s0_sel_src.elemIdx 796 s0_out.elemIdxInsideVd := s0_sel_src.elemIdxInsideVd 797 s0_out.alignedType := s0_sel_src.alignedType 798 s0_out.mbIndex := s0_sel_src.mbIndex 799 s0_out.vecBaseVaddr := s0_sel_src.vecBaseVaddr 800 // s0_out.flowPtr := s0_sel_src.flowPtr 801 s0_out.uop.exceptionVec(loadAddrMisaligned) := (!s0_addr_aligned || s0_sel_src.uop.exceptionVec(loadAddrMisaligned)) && s0_sel_src.vecActive && !s0_misalignWith16Byte 802 s0_out.isMisalign := (!s0_addr_aligned || s0_sel_src.uop.exceptionVec(loadAddrMisaligned)) && s0_sel_src.vecActive 803 s0_out.forward_tlDchannel := s0_src_select_vec(super_rep_idx) 804 when(io.tlb.req.valid && s0_sel_src.isFirstIssue) { 805 s0_out.uop.debugInfo.tlbFirstReqTime := GTimer() 806 }.otherwise{ 807 s0_out.uop.debugInfo.tlbFirstReqTime := s0_sel_src.uop.debugInfo.tlbFirstReqTime 808 } 809 s0_out.schedIndex := s0_sel_src.sched_idx 810 //for Svpbmt Nc 811 s0_out.nc := s0_sel_src.isnc 812 s0_out.data := s0_sel_src.data 813 s0_out.misalignWith16Byte := s0_misalignWith16Byte 814 s0_out.misalignNeedWakeUp := s0_misalignNeedWakeUp 815 s0_out.isFinalSplit := s0_finalSplit 816 817 // load fast replay 818 io.fast_rep_in.ready := (s0_can_go && io.dcache.req.ready && s0_src_ready_vec(fast_rep_idx)) 819 820 // mmio 821 io.lsq.uncache.ready := s0_mmio_fire 822 io.lsq.nc_ldin.ready := s0_src_ready_vec(nc_idx) && s0_can_go 823 824 // load flow source ready 825 // cache missed load has highest priority 826 // always accept cache missed load flow from load replay queue 827 io.replay.ready := (s0_can_go && io.dcache.req.ready && (s0_src_ready_vec(lsq_rep_idx) && !s0_rep_stall || s0_src_select_vec(super_rep_idx))) 828 829 // accept load flow from rs when: 830 // 1) there is no lsq-replayed load 831 // 2) there is no fast replayed load 832 // 3) there is no high confidence prefetch request 833 io.vecldin.ready := s0_can_go && io.dcache.req.ready && s0_src_ready_vec(vec_iss_idx) 834 io.ldin.ready := s0_can_go && io.dcache.req.ready && s0_src_ready_vec(int_iss_idx) 835 io.misalign_ldin.ready := s0_can_go && io.dcache.req.ready && s0_src_ready_vec(mab_idx) 836 837 // for hw prefetch load flow feedback, to be added later 838 // io.prefetch_in.ready := s0_hw_prf_select 839 840 // dcache replacement extra info 841 // TODO: should prefetch load update replacement? 842 io.dcache.replacementUpdated := Mux(s0_src_select_vec(lsq_rep_idx) || s0_src_select_vec(super_rep_idx), io.replay.bits.replacementUpdated, false.B) 843 844 // load wakeup 845 // TODO: vector load wakeup? frm_mabuf wakeup? 846 val s0_wakeup_selector = Seq( 847 s0_misalign_wakeup_fire, 848 s0_src_valid_vec(super_rep_idx), 849 s0_src_valid_vec(fast_rep_idx), 850 s0_mmio_fire, 851 s0_nc_fire, 852 s0_src_valid_vec(lsq_rep_idx), 853 s0_src_valid_vec(int_iss_idx) 854 ) 855 val s0_wakeup_format = Seq( 856 io.misalign_ldin.bits.uop, 857 io.replay.bits.uop, 858 io.fast_rep_in.bits.uop, 859 io.lsq.uncache.bits.uop, 860 io.lsq.nc_ldin.bits.uop, 861 io.replay.bits.uop, 862 io.ldin.bits.uop, 863 ) 864 val s0_wakeup_uop = ParallelPriorityMux(s0_wakeup_selector, s0_wakeup_format) 865 io.wakeup.valid := s0_fire && !s0_sel_src.isvec && !s0_sel_src.frm_mabuf && ( 866 s0_src_valid_vec(super_rep_idx) || 867 s0_src_valid_vec(fast_rep_idx) || 868 s0_src_valid_vec(lsq_rep_idx) || 869 (s0_src_valid_vec(int_iss_idx) && !s0_sel_src.prf && 870 !s0_src_valid_vec(vec_iss_idx) && !s0_src_valid_vec(high_pf_idx)) 871 ) || s0_mmio_fire || s0_nc_fire || s0_misalign_wakeup_fire 872 io.wakeup.bits := s0_wakeup_uop 873 874 // prefetch.i(Zicbop) 875 io.ifetchPrefetch.valid := RegNext(s0_src_select_vec(int_iss_idx) && s0_sel_src.prf_i) 876 io.ifetchPrefetch.bits.vaddr := RegEnable(s0_out.vaddr, 0.U, s0_src_select_vec(int_iss_idx) && s0_sel_src.prf_i) 877 878 XSDebug(io.dcache.req.fire, 879 p"[DCACHE LOAD REQ] pc ${Hexadecimal(s0_sel_src.uop.pc)}, vaddr ${Hexadecimal(s0_dcache_vaddr)}\n" 880 ) 881 XSDebug(s0_valid, 882 p"S0: pc ${Hexadecimal(s0_out.uop.pc)}, lId ${Hexadecimal(s0_out.uop.lqIdx.asUInt)}, " + 883 p"vaddr ${Hexadecimal(s0_out.vaddr)}, mask ${Hexadecimal(s0_out.mask)}\n") 884 885 // Pipeline 886 // -------------------------------------------------------------------------------- 887 // stage 1 888 // -------------------------------------------------------------------------------- 889 // TLB resp (send paddr to dcache) 890 val s1_valid = RegInit(false.B) 891 val s1_in = Wire(new LqWriteBundle) 892 val s1_out = Wire(new LqWriteBundle) 893 val s1_kill = Wire(Bool()) 894 val s1_can_go = s2_ready 895 val s1_fire = s1_valid && !s1_kill && s1_can_go 896 val s1_vecActive = RegEnable(s0_out.vecActive, true.B, s0_fire) 897 val s1_nc_with_data = RegNext(s0_nc_with_data) 898 899 s1_ready := !s1_valid || s1_kill || s2_ready 900 when (s0_fire) { s1_valid := true.B } 901 .elsewhen (s1_fire) { s1_valid := false.B } 902 .elsewhen (s1_kill) { s1_valid := false.B } 903 s1_in := RegEnable(s0_out, s0_fire) 904 905 val s1_fast_rep_dly_kill = RegEnable(io.fast_rep_in.bits.lateKill, io.fast_rep_in.valid) && s1_in.isFastReplay 906 val s1_fast_rep_dly_err = RegEnable(io.fast_rep_in.bits.delayedLoadError, io.fast_rep_in.valid) && s1_in.isFastReplay 907 val s1_l2l_fwd_dly_err = RegEnable(io.l2l_fwd_in.dly_ld_err, io.l2l_fwd_in.valid) && s1_in.isFastPath 908 val s1_dly_err = s1_fast_rep_dly_err || s1_l2l_fwd_dly_err 909 val s1_vaddr_hi = Wire(UInt()) 910 val s1_vaddr_lo = Wire(UInt()) 911 val s1_vaddr = Wire(UInt()) 912 val s1_paddr_dup_lsu = Wire(UInt()) 913 val s1_gpaddr_dup_lsu = Wire(UInt()) 914 val s1_paddr_dup_dcache = Wire(UInt()) 915 val s1_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, LduCfg).asUInt.orR // af & pf exception were modified below. 916 val s1_tlb_miss = io.tlb.resp.bits.miss && io.tlb.resp.valid && s1_valid 917 val s1_tlb_fast_miss = io.tlb.resp.bits.fastMiss && io.tlb.resp.valid && s1_valid 918 val s1_pbmt = Mux(!s1_tlb_miss, io.tlb.resp.bits.pbmt.head, 0.U(Pbmt.width.W)) 919 val s1_nc = s1_in.nc 920 val s1_prf = s1_in.isPrefetch 921 val s1_hw_prf = s1_in.isHWPrefetch 922 val s1_sw_prf = s1_prf && !s1_hw_prf 923 val s1_tlb_memidx = io.tlb.resp.bits.memidx 924 925 s1_vaddr_hi := s1_in.vaddr(VAddrBits - 1, 6) 926 s1_vaddr_lo := s1_in.vaddr(5, 0) 927 s1_vaddr := Cat(s1_vaddr_hi, s1_vaddr_lo) 928 s1_paddr_dup_lsu := Mux(s1_in.tlbNoQuery, s1_in.paddr, io.tlb.resp.bits.paddr(0)) 929 s1_paddr_dup_dcache := Mux(s1_in.tlbNoQuery, s1_in.paddr, io.tlb.resp.bits.paddr(1)) 930 s1_gpaddr_dup_lsu := Mux(s1_in.isFastReplay, s1_in.paddr, io.tlb.resp.bits.gpaddr(0)) 931 932 when (s1_tlb_memidx.is_ld && io.tlb.resp.valid && !s1_tlb_miss && s1_tlb_memidx.idx === s1_in.uop.lqIdx.value) { 933 // printf("load idx = %d\n", s1_tlb_memidx.idx) 934 s1_out.uop.debugInfo.tlbRespTime := GTimer() 935 } 936 937 io.tlb.req_kill := s1_kill || s1_dly_err 938 io.tlb.req.bits.pmp_addr := s1_in.paddr 939 io.tlb.resp.ready := true.B 940 941 io.dcache.s1_paddr_dup_lsu <> s1_paddr_dup_lsu 942 io.dcache.s1_paddr_dup_dcache <> s1_paddr_dup_dcache 943 io.dcache.s1_kill := s1_kill || s1_dly_err || s1_tlb_miss || s1_exception 944 io.dcache.s1_kill_data_read := s1_kill || s1_dly_err || s1_tlb_fast_miss 945 946 // store to load forwarding 947 io.sbuffer.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_dly_err || s1_prf) 948 io.sbuffer.vaddr := s1_vaddr 949 io.sbuffer.paddr := s1_paddr_dup_lsu 950 io.sbuffer.uop := s1_in.uop 951 io.sbuffer.sqIdx := s1_in.uop.sqIdx 952 io.sbuffer.mask := s1_in.mask 953 io.sbuffer.pc := s1_in.uop.pc // FIXME: remove it 954 955 io.ubuffer.valid := s1_valid && s1_nc_with_data && !(s1_exception || s1_tlb_miss || s1_kill || s1_dly_err || s1_prf) 956 io.ubuffer.vaddr := s1_vaddr 957 io.ubuffer.paddr := s1_paddr_dup_lsu 958 io.ubuffer.uop := s1_in.uop 959 io.ubuffer.sqIdx := s1_in.uop.sqIdx 960 io.ubuffer.mask := s1_in.mask 961 io.ubuffer.pc := s1_in.uop.pc // FIXME: remove it 962 963 io.lsq.forward.valid := s1_valid && !(s1_exception || s1_tlb_miss || s1_kill || s1_dly_err || s1_prf) 964 io.lsq.forward.vaddr := s1_vaddr 965 io.lsq.forward.paddr := s1_paddr_dup_lsu 966 io.lsq.forward.uop := s1_in.uop 967 io.lsq.forward.sqIdx := s1_in.uop.sqIdx 968 io.lsq.forward.sqIdxMask := 0.U 969 io.lsq.forward.mask := s1_in.mask 970 io.lsq.forward.pc := s1_in.uop.pc // FIXME: remove it 971 972 // st-ld violation query 973 // if store unit is 128-bits memory access, need match 128-bit 974 private val s1_isMatch128 = io.stld_nuke_query.map(x => (x.bits.matchLine || ((s1_in.isvec || s1_in.misalignWith16Byte) && s1_in.is128bit))) 975 val s1_nuke_paddr_match = VecInit((0 until StorePipelineWidth).zip(s1_isMatch128).map{case (w, s) => {Mux(s, 976 s1_paddr_dup_lsu(PAddrBits-1, 4) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 4), 977 s1_paddr_dup_lsu(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3))}}) 978 val s1_nuke = VecInit((0 until StorePipelineWidth).map(w => { 979 io.stld_nuke_query(w).valid && // query valid 980 isAfter(s1_in.uop.robIdx, io.stld_nuke_query(w).bits.robIdx) && // older store 981 s1_nuke_paddr_match(w) && // paddr match 982 (s1_in.mask & io.stld_nuke_query(w).bits.mask).orR // data mask contain 983 })).asUInt.orR && !s1_tlb_miss 984 985 s1_out := s1_in 986 s1_out.vaddr := s1_vaddr 987 s1_out.fullva := io.tlb.resp.bits.fullva 988 s1_out.vaNeedExt := io.tlb.resp.bits.excp(0).vaNeedExt 989 s1_out.isHyper := io.tlb.resp.bits.excp(0).isHyper 990 s1_out.paddr := s1_paddr_dup_lsu 991 s1_out.gpaddr := s1_gpaddr_dup_lsu 992 s1_out.isForVSnonLeafPTE := io.tlb.resp.bits.isForVSnonLeafPTE 993 s1_out.tlbMiss := s1_tlb_miss 994 s1_out.ptwBack := io.tlb.resp.bits.ptwBack 995 s1_out.rep_info.debug := s1_in.uop.debugInfo 996 s1_out.rep_info.nuke := s1_nuke && !s1_sw_prf 997 s1_out.delayedLoadError := s1_dly_err 998 s1_out.nc := s1_nc || Pbmt.isNC(s1_pbmt) 999 s1_out.mmio := Pbmt.isIO(s1_pbmt) 1000 1001 when (!s1_dly_err) { 1002 // current ori test will cause the case of ldest == 0, below will be modifeid in the future. 1003 // af & pf exception were modified 1004 // if is tlbNoQuery request, don't trigger exception from tlb resp 1005 s1_out.uop.exceptionVec(loadPageFault) := io.tlb.resp.bits.excp(0).pf.ld && s1_vecActive && !s1_tlb_miss && !s1_in.tlbNoQuery 1006 s1_out.uop.exceptionVec(loadGuestPageFault) := io.tlb.resp.bits.excp(0).gpf.ld && !s1_tlb_miss && !s1_in.tlbNoQuery 1007 s1_out.uop.exceptionVec(loadAccessFault) := io.tlb.resp.bits.excp(0).af.ld && s1_vecActive && !s1_tlb_miss && !s1_in.tlbNoQuery 1008 when (RegNext(io.tlb.req.bits.checkfullva) && 1009 (s1_out.uop.exceptionVec(loadPageFault) || 1010 s1_out.uop.exceptionVec(loadGuestPageFault) || 1011 s1_out.uop.exceptionVec(loadAccessFault))) { 1012 s1_out.uop.exceptionVec(loadAddrMisaligned) := false.B 1013 s1_out.isMisalign := false.B 1014 } 1015 } .otherwise { 1016 s1_out.uop.exceptionVec(loadPageFault) := false.B 1017 s1_out.uop.exceptionVec(loadGuestPageFault) := false.B 1018 s1_out.uop.exceptionVec(loadAddrMisaligned) := false.B 1019 s1_out.isMisalign := false.B 1020 s1_out.uop.exceptionVec(loadAccessFault) := s1_dly_err && s1_vecActive 1021 } 1022 1023 // pointer chasing 1024 val s1_try_ptr_chasing = GatedValidRegNext(s0_do_try_ptr_chasing, false.B) 1025 val s1_ptr_chasing_vaddr = RegEnable(s0_ptr_chasing_vaddr, s0_do_try_ptr_chasing) 1026 val s1_fu_op_type_not_ld = WireInit(false.B) 1027 val s1_not_fast_match = WireInit(false.B) 1028 val s1_addr_mismatch = WireInit(false.B) 1029 val s1_addr_misaligned = WireInit(false.B) 1030 val s1_fast_mismatch = WireInit(false.B) 1031 val s1_ptr_chasing_canceled = WireInit(false.B) 1032 val s1_cancel_ptr_chasing = WireInit(false.B) 1033 1034 val s1_redirect_reg = Wire(Valid(new Redirect)) 1035 s1_redirect_reg.bits := RegEnable(io.redirect.bits, io.redirect.valid) 1036 s1_redirect_reg.valid := GatedValidRegNext(io.redirect.valid) 1037 1038 s1_kill := s1_fast_rep_dly_kill || 1039 s1_cancel_ptr_chasing || 1040 s1_in.uop.robIdx.needFlush(io.redirect) || 1041 (s1_in.uop.robIdx.needFlush(s1_redirect_reg) && !GatedValidRegNext(s0_try_ptr_chasing)) || 1042 RegEnable(s0_kill, false.B, io.ldin.valid || 1043 io.vecldin.valid || io.replay.valid || 1044 io.l2l_fwd_in.valid || io.fast_rep_in.valid || 1045 io.misalign_ldin.valid || io.lsq.nc_ldin.valid 1046 ) 1047 1048 if (EnableLoadToLoadForward) { 1049 // Sometimes, we need to cancel the load-load forwarding. 1050 // These can be put at S0 if timing is bad at S1. 1051 // Case 0: CACHE_SET(base + offset) != CACHE_SET(base) (lowest 6-bit addition has an overflow) 1052 s1_addr_mismatch := s1_ptr_chasing_vaddr(6) || 1053 RegEnable(io.ld_fast_imm(11, 6).orR, s0_do_try_ptr_chasing) 1054 // Case 1: the address is not 64-bit aligned or the fuOpType is not LD 1055 s1_addr_misaligned := s1_ptr_chasing_vaddr(2, 0).orR 1056 s1_fu_op_type_not_ld := io.ldin.bits.uop.fuOpType =/= LSUOpType.ld 1057 // Case 2: this load-load uop is cancelled 1058 s1_ptr_chasing_canceled := !io.ldin.valid 1059 // Case 3: fast mismatch 1060 s1_fast_mismatch := RegEnable(!io.ld_fast_match, s0_do_try_ptr_chasing) 1061 1062 when (s1_try_ptr_chasing) { 1063 s1_cancel_ptr_chasing := s1_addr_mismatch || 1064 s1_addr_misaligned || 1065 s1_fu_op_type_not_ld || 1066 s1_ptr_chasing_canceled || 1067 s1_fast_mismatch 1068 1069 s1_in.uop := io.ldin.bits.uop 1070 s1_in.isFirstIssue := io.ldin.bits.isFirstIssue 1071 s1_vaddr_lo := s1_ptr_chasing_vaddr(5, 0) 1072 s1_paddr_dup_lsu := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 1073 s1_paddr_dup_dcache := Cat(io.tlb.resp.bits.paddr(0)(PAddrBits - 1, 6), s1_vaddr_lo) 1074 1075 // recored tlb time when get the data to ensure the correctness of the latency calculation (although it should not record in here, because it does not use tlb) 1076 s1_in.uop.debugInfo.tlbFirstReqTime := GTimer() 1077 s1_in.uop.debugInfo.tlbRespTime := GTimer() 1078 } 1079 when (!s1_cancel_ptr_chasing) { 1080 s0_ptr_chasing_canceled := s1_try_ptr_chasing && 1081 !io.replay.fire && !io.fast_rep_in.fire && 1082 !(s0_src_valid_vec(high_pf_idx) && io.canAcceptHighConfPrefetch) && 1083 !io.misalign_ldin.fire && 1084 !io.lsq.nc_ldin.valid 1085 when (s1_try_ptr_chasing) { 1086 io.ldin.ready := true.B 1087 } 1088 } 1089 } 1090 1091 // pre-calcuate sqIdx mask in s0, then send it to lsq in s1 for forwarding 1092 val s1_sqIdx_mask = RegEnable(UIntToMask(s0_out.uop.sqIdx.value, StoreQueueSize), s0_fire) 1093 // to enable load-load, sqIdxMask must be calculated based on ldin.uop 1094 // If the timing here is not OK, load-load forwarding has to be disabled. 1095 // Or we calculate sqIdxMask at RS?? 1096 io.lsq.forward.sqIdxMask := s1_sqIdx_mask 1097 if (EnableLoadToLoadForward) { 1098 when (s1_try_ptr_chasing) { 1099 io.lsq.forward.sqIdxMask := UIntToMask(io.ldin.bits.uop.sqIdx.value, StoreQueueSize) 1100 } 1101 } 1102 1103 io.forward_mshr.valid := s1_valid && s1_out.forward_tlDchannel 1104 io.forward_mshr.mshrid := s1_out.mshrid 1105 io.forward_mshr.paddr := s1_out.paddr 1106 1107 val loadTrigger = Module(new MemTrigger(MemType.LOAD)) 1108 loadTrigger.io.fromCsrTrigger.tdataVec := io.fromCsrTrigger.tdataVec 1109 loadTrigger.io.fromCsrTrigger.tEnableVec := io.fromCsrTrigger.tEnableVec 1110 loadTrigger.io.fromCsrTrigger.triggerCanRaiseBpExp := io.fromCsrTrigger.triggerCanRaiseBpExp 1111 loadTrigger.io.fromCsrTrigger.debugMode := io.fromCsrTrigger.debugMode 1112 loadTrigger.io.fromLoadStore.vaddr := s1_vaddr 1113 loadTrigger.io.fromLoadStore.isVectorUnitStride := s1_in.isvec && s1_in.is128bit 1114 loadTrigger.io.fromLoadStore.mask := s1_in.mask 1115 1116 val s1_trigger_action = loadTrigger.io.toLoadStore.triggerAction 1117 val s1_trigger_debug_mode = TriggerAction.isDmode(s1_trigger_action) 1118 val s1_trigger_breakpoint = TriggerAction.isExp(s1_trigger_action) 1119 s1_out.uop.trigger := s1_trigger_action 1120 s1_out.uop.exceptionVec(breakPoint) := s1_trigger_breakpoint 1121 s1_out.vecVaddrOffset := Mux( 1122 s1_trigger_debug_mode || s1_trigger_breakpoint, 1123 loadTrigger.io.toLoadStore.triggerVaddr - s1_in.vecBaseVaddr, 1124 s1_in.vaddr + genVFirstUnmask(s1_in.mask).asUInt - s1_in.vecBaseVaddr 1125 ) 1126 s1_out.vecTriggerMask := Mux(s1_trigger_debug_mode || s1_trigger_breakpoint, loadTrigger.io.toLoadStore.triggerMask, 0.U) 1127 1128 XSDebug(s1_valid, 1129 p"S1: pc ${Hexadecimal(s1_out.uop.pc)}, lId ${Hexadecimal(s1_out.uop.lqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 1130 p"paddr ${Hexadecimal(s1_out.paddr)}, mmio ${s1_out.mmio}\n") 1131 1132 // Pipeline 1133 // -------------------------------------------------------------------------------- 1134 // stage 2 1135 // -------------------------------------------------------------------------------- 1136 // s2: DCache resp 1137 val s2_valid = RegInit(false.B) 1138 val s2_in = Wire(new LqWriteBundle) 1139 val s2_out = Wire(new LqWriteBundle) 1140 val s2_kill = Wire(Bool()) 1141 val s2_can_go = s3_ready 1142 val s2_fire = s2_valid && !s2_kill && s2_can_go 1143 val s2_vecActive = RegEnable(s1_out.vecActive, true.B, s1_fire) 1144 val s2_isvec = RegEnable(s1_out.isvec, false.B, s1_fire) 1145 val s2_data_select = genRdataOH(s2_out.uop) 1146 val s2_data_select_by_offset = genDataSelectByOffset(s2_out.paddr(3, 0)) 1147 val s2_frm_mabuf = s2_in.isFrmMisAlignBuf 1148 val s2_pbmt = RegEnable(s1_pbmt, s1_fire) 1149 val s2_trigger_debug_mode = RegEnable(s1_trigger_debug_mode, false.B, s1_fire) 1150 val s2_nc_with_data = RegNext(s1_nc_with_data) 1151 val s2_mmio_req = Wire(Valid(new MemExuOutput)) 1152 s2_mmio_req.valid := RegNextN(io.lsq.uncache.fire, 2, Some(false.B)) 1153 s2_mmio_req.bits := RegNextN(io.lsq.uncache.bits, 2) 1154 1155 s2_kill := s2_in.uop.robIdx.needFlush(io.redirect) 1156 s2_ready := !s2_valid || s2_kill || s3_ready 1157 when (s1_fire) { s2_valid := true.B } 1158 .elsewhen (s2_fire) { s2_valid := false.B } 1159 .elsewhen (s2_kill) { s2_valid := false.B } 1160 s2_in := RegEnable(s1_out, s1_fire) 1161 1162 val s2_pmp = WireInit(io.pmp) 1163 val s2_isMisalign = WireInit(s2_in.isMisalign) 1164 1165 val s2_prf = s2_in.isPrefetch 1166 val s2_hw_prf = s2_in.isHWPrefetch 1167 val s2_exception_vec = WireInit(s2_in.uop.exceptionVec) 1168 val s2_un_misalign_exception = s2_vecActive && 1169 (s2_trigger_debug_mode || ExceptionNO.selectByFuAndUnSelect(s2_exception_vec, LduCfg, Seq(loadAddrMisaligned)).asUInt.orR) 1170 val s2_check_mmio = !s2_prf && !s2_in.tlbMiss && Mux(Pbmt.isUncache(s2_pbmt), s2_in.mmio, s2_pmp.mmio) && !s2_un_misalign_exception 1171 // exception that may cause load addr to be invalid / illegal 1172 // if such exception happen, that inst and its exception info 1173 // will be force writebacked to rob 1174 val s2_actually_uncache = Pbmt.isPMA(s2_pbmt) && s2_pmp.mmio || s2_in.nc || s2_in.mmio 1175 val s2_memBackTypeMM = !s2_pmp.mmio 1176 when (!s2_in.delayedLoadError) { 1177 s2_exception_vec(loadAccessFault) := s2_vecActive && ( 1178 s2_in.uop.exceptionVec(loadAccessFault) || 1179 s2_pmp.ld || 1180 (s2_isvec || s2_frm_mabuf) && s2_actually_uncache && !s2_prf && !s2_in.tlbMiss || 1181 io.dcache.resp.bits.tag_error && GatedValidRegNext(io.csrCtrl.cache_error_enable) 1182 ) 1183 } 1184 1185 // soft prefetch will not trigger any exception (but ecc error interrupt may 1186 // be triggered) 1187 val s2_tlb_unrelated_exceps = s2_in.uop.exceptionVec(loadAddrMisaligned) || 1188 s2_in.uop.exceptionVec(breakPoint) 1189 when (!s2_in.delayedLoadError && (s2_prf || s2_in.tlbMiss && !s2_tlb_unrelated_exceps)) { 1190 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 1191 s2_isMisalign := false.B 1192 } 1193 val s2_exception = s2_vecActive && 1194 (s2_trigger_debug_mode || ExceptionNO.selectByFu(s2_exception_vec, LduCfg).asUInt.orR) 1195 val s2_mis_align = s2_valid && GatedValidRegNext(io.csrCtrl.hd_misalign_ld_enable) && 1196 s2_out.isMisalign && !s2_in.misalignWith16Byte && !s2_exception_vec(breakPoint) && !s2_trigger_debug_mode && !s2_check_mmio 1197 val (s2_fwd_frm_d_chan, s2_fwd_data_frm_d_chan, s2_d_corrupt) = io.tl_d_channel.forward(s1_valid && s1_out.forward_tlDchannel, s1_out.mshrid, s1_out.paddr) 1198 val (s2_fwd_data_valid, s2_fwd_frm_mshr, s2_fwd_data_frm_mshr, s2_mshr_corrupt) = io.forward_mshr.forward() 1199 val s2_fwd_frm_d_chan_or_mshr = s2_fwd_data_valid && (s2_fwd_frm_d_chan || s2_fwd_frm_mshr) 1200 1201 // writeback access fault caused by ecc error / bus error 1202 // * ecc data error is slow to generate, so we will not use it until load stage 3 1203 // * in load stage 3, an extra signal io.load_error will be used to 1204 // * if pbmt =/= 0, mmio is up to pbmt; otherwise, it's up to pmp 1205 val s2_mmio = !s2_prf && 1206 !s2_exception && !s2_in.tlbMiss && 1207 Mux(Pbmt.isUncache(s2_pbmt), s2_in.mmio, s2_pmp.mmio) 1208 val s2_uncache = !s2_prf && !s2_exception && !s2_in.tlbMiss && s2_actually_uncache 1209 1210 val s2_full_fwd = Wire(Bool()) 1211 val s2_mem_amb = s2_in.uop.storeSetHit && 1212 io.lsq.forward.addrInvalid && RegNext(io.lsq.forward.valid) 1213 1214 val s2_tlb_miss = s2_in.tlbMiss 1215 val s2_fwd_fail = io.lsq.forward.dataInvalid && RegNext(io.lsq.forward.valid) 1216 val s2_dcache_miss = io.dcache.resp.bits.miss && 1217 !s2_fwd_frm_d_chan_or_mshr && 1218 !s2_full_fwd && !s2_in.nc 1219 1220 val s2_mq_nack = io.dcache.s2_mq_nack && 1221 !s2_fwd_frm_d_chan_or_mshr && 1222 !s2_full_fwd && !s2_in.nc 1223 1224 val s2_bank_conflict = io.dcache.s2_bank_conflict && 1225 !s2_fwd_frm_d_chan_or_mshr && 1226 !s2_full_fwd && !s2_in.nc 1227 1228 val s2_wpu_pred_fail = io.dcache.s2_wpu_pred_fail && 1229 !s2_fwd_frm_d_chan_or_mshr && 1230 !s2_full_fwd && !s2_in.nc 1231 1232 val s2_rar_nack = io.lsq.ldld_nuke_query.req.valid && 1233 !io.lsq.ldld_nuke_query.req.ready 1234 1235 val s2_raw_nack = io.lsq.stld_nuke_query.req.valid && 1236 !io.lsq.stld_nuke_query.req.ready 1237 // st-ld violation query 1238 // NeedFastRecovery Valid when 1239 // 1. Fast recovery query request Valid. 1240 // 2. Load instruction is younger than requestors(store instructions). 1241 // 3. Physical address match. 1242 // 4. Data contains. 1243 private val s2_isMatch128 = io.stld_nuke_query.map(x => (x.bits.matchLine || ((s2_in.isvec || s2_in.misalignWith16Byte) && s2_in.is128bit))) 1244 val s2_nuke_paddr_match = VecInit((0 until StorePipelineWidth).zip(s2_isMatch128).map{case (w, s) => {Mux(s, 1245 s2_in.paddr(PAddrBits-1, 4) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 4), 1246 s2_in.paddr(PAddrBits-1, 3) === io.stld_nuke_query(w).bits.paddr(PAddrBits-1, 3))}}) 1247 val s2_nuke = VecInit((0 until StorePipelineWidth).map(w => { 1248 io.stld_nuke_query(w).valid && // query valid 1249 isAfter(s2_in.uop.robIdx, io.stld_nuke_query(w).bits.robIdx) && // older store 1250 s2_nuke_paddr_match(w) && // paddr match 1251 (s2_in.mask & io.stld_nuke_query(w).bits.mask).orR // data mask contain 1252 })).asUInt.orR && !s2_tlb_miss || s2_in.rep_info.nuke 1253 1254 val s2_cache_handled = io.dcache.resp.bits.handled 1255 1256 //if it is NC with data, it should handle the replayed situation. 1257 //else s2_uncache will enter uncache buffer. 1258 val s2_troublem = !s2_exception && 1259 (!s2_uncache || s2_nc_with_data) && 1260 !s2_prf && 1261 !s2_in.delayedLoadError 1262 1263 io.dcache.resp.ready := true.B 1264 val s2_dcache_should_resp = !(s2_in.tlbMiss || s2_exception || s2_in.delayedLoadError || s2_uncache || s2_prf) 1265 assert(!(s2_valid && (s2_dcache_should_resp && !io.dcache.resp.valid)), "DCache response got lost") 1266 1267 // fast replay require 1268 val s2_dcache_fast_rep = (s2_mq_nack || !s2_dcache_miss && (s2_bank_conflict || s2_wpu_pred_fail)) 1269 val s2_nuke_fast_rep = !s2_mq_nack && 1270 !s2_dcache_miss && 1271 !s2_bank_conflict && 1272 !s2_wpu_pred_fail && 1273 s2_nuke 1274 1275 val s2_fast_rep = !s2_in.isFastReplay && 1276 !s2_mem_amb && 1277 !s2_tlb_miss && 1278 !s2_fwd_fail && 1279 (s2_dcache_fast_rep || s2_nuke_fast_rep) && 1280 s2_troublem && 1281 !s2_in.misalignNeedWakeUp 1282 1283 // need allocate new entry 1284 val s2_can_query = !s2_mem_amb && 1285 !s2_tlb_miss && 1286 !s2_fwd_fail && 1287 !s2_frm_mabuf && 1288 !s2_fast_rep && 1289 s2_troublem 1290 1291 val s2_data_fwded = s2_dcache_miss && s2_full_fwd 1292 1293 // For misaligned, we will keep the misaligned exception at S2 and before. 1294 // Here a judgement is made as to whether a misaligned exception needs to actually be generated. 1295 // We will generate misaligned exceptions at mmio. 1296 val s2_real_exceptionVec = WireInit(s2_exception_vec) 1297 s2_real_exceptionVec(loadAddrMisaligned) := s2_out.isMisalign && s2_check_mmio 1298 s2_real_exceptionVec(loadAccessFault) := s2_exception_vec(loadAccessFault) || 1299 s2_fwd_frm_d_chan && s2_d_corrupt || 1300 s2_fwd_frm_mshr && s2_mshr_corrupt 1301 val s2_real_exception = s2_vecActive && 1302 (s2_trigger_debug_mode || ExceptionNO.selectByFu(s2_real_exceptionVec, LduCfg).asUInt.orR) 1303 1304 val s2_fwd_vp_match_invalid = io.lsq.forward.matchInvalid || io.sbuffer.matchInvalid || io.ubuffer.matchInvalid 1305 val s2_vp_match_fail = s2_fwd_vp_match_invalid && s2_troublem 1306 val s2_safe_wakeup = !s2_out.rep_info.need_rep && !s2_mmio && (!s2_in.nc || s2_nc_with_data) && !s2_mis_align && !s2_real_exception || s2_in.misalignNeedWakeUp // don't need to replay and is not a mmio\misalign no data 1307 val s2_safe_writeback = s2_real_exception || s2_safe_wakeup || s2_vp_match_fail || s2_in.misalignNeedWakeUp 1308 1309 // ld-ld violation require 1310 io.lsq.ldld_nuke_query.req.valid := s2_valid && s2_can_query 1311 io.lsq.ldld_nuke_query.req.bits.uop := s2_in.uop 1312 io.lsq.ldld_nuke_query.req.bits.mask := s2_in.mask 1313 io.lsq.ldld_nuke_query.req.bits.paddr := s2_in.paddr 1314 io.lsq.ldld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid || s2_nc_with_data, true.B, !s2_dcache_miss) 1315 io.lsq.ldld_nuke_query.req.bits.is_nc := s2_nc_with_data 1316 1317 // st-ld violation require 1318 io.lsq.stld_nuke_query.req.valid := s2_valid && s2_can_query 1319 io.lsq.stld_nuke_query.req.bits.uop := s2_in.uop 1320 io.lsq.stld_nuke_query.req.bits.mask := s2_in.mask 1321 io.lsq.stld_nuke_query.req.bits.paddr := s2_in.paddr 1322 io.lsq.stld_nuke_query.req.bits.data_valid := Mux(s2_full_fwd || s2_fwd_data_valid || s2_nc_with_data, true.B, !s2_dcache_miss) 1323 io.lsq.stld_nuke_query.req.bits.is_nc := s2_nc_with_data 1324 1325 // merge forward result 1326 // lsq has higher priority than sbuffer 1327 val s2_fwd_mask = Wire(Vec((VLEN/8), Bool())) 1328 val s2_fwd_data = Wire(Vec((VLEN/8), UInt(8.W))) 1329 s2_full_fwd := ((~s2_fwd_mask.asUInt).asUInt & s2_in.mask) === 0.U && !io.lsq.forward.dataInvalid 1330 // generate XLEN/8 Muxs 1331 for (i <- 0 until VLEN / 8) { 1332 s2_fwd_mask(i) := io.lsq.forward.forwardMask(i) || io.sbuffer.forwardMask(i) || io.ubuffer.forwardMask(i) 1333 s2_fwd_data(i) := 1334 Mux(io.lsq.forward.forwardMask(i), io.lsq.forward.forwardData(i), 1335 Mux(s2_nc_with_data, io.ubuffer.forwardData(i), 1336 io.sbuffer.forwardData(i))) 1337 } 1338 1339 XSDebug(s2_fire, "[FWD LOAD RESP] pc %x fwd %x(%b) + %x(%b)\n", 1340 s2_in.uop.pc, 1341 io.lsq.forward.forwardData.asUInt, io.lsq.forward.forwardMask.asUInt, 1342 s2_in.forwardData.asUInt, s2_in.forwardMask.asUInt 1343 ) 1344 1345 // 1346 s2_out := s2_in 1347 s2_out.uop.fpWen := s2_in.uop.fpWen 1348 s2_out.nc := s2_in.nc 1349 s2_out.mmio := s2_mmio 1350 s2_out.memBackTypeMM := s2_memBackTypeMM 1351 s2_out.isMisalign := s2_isMisalign 1352 s2_out.uop.flushPipe := false.B 1353 s2_out.uop.exceptionVec := s2_real_exceptionVec 1354 s2_out.forwardMask := s2_fwd_mask 1355 s2_out.forwardData := s2_fwd_data 1356 s2_out.handledByMSHR := s2_cache_handled 1357 s2_out.miss := s2_dcache_miss && s2_troublem 1358 s2_out.feedbacked := io.feedback_fast.valid 1359 s2_out.uop.vpu.vstart := Mux(s2_in.isLoadReplay || s2_in.isFastReplay, s2_in.uop.vpu.vstart, s2_in.vecVaddrOffset >> s2_in.uop.vpu.veew) 1360 1361 // Generate replay signal caused by: 1362 // * st-ld violation check 1363 // * tlb miss 1364 // * dcache replay 1365 // * forward data invalid 1366 // * dcache miss 1367 s2_out.rep_info.mem_amb := s2_mem_amb && s2_troublem 1368 s2_out.rep_info.tlb_miss := s2_tlb_miss && s2_troublem 1369 s2_out.rep_info.fwd_fail := s2_fwd_fail && s2_troublem 1370 s2_out.rep_info.dcache_rep := s2_mq_nack && s2_troublem 1371 s2_out.rep_info.dcache_miss := s2_dcache_miss && s2_troublem 1372 s2_out.rep_info.bank_conflict := s2_bank_conflict && s2_troublem 1373 s2_out.rep_info.wpu_fail := s2_wpu_pred_fail && s2_troublem 1374 s2_out.rep_info.rar_nack := s2_rar_nack && s2_troublem 1375 s2_out.rep_info.raw_nack := s2_raw_nack && s2_troublem 1376 s2_out.rep_info.nuke := s2_nuke && s2_troublem 1377 s2_out.rep_info.full_fwd := s2_data_fwded 1378 s2_out.rep_info.data_inv_sq_idx := io.lsq.forward.dataInvalidSqIdx 1379 s2_out.rep_info.addr_inv_sq_idx := io.lsq.forward.addrInvalidSqIdx 1380 s2_out.rep_info.rep_carry := io.dcache.resp.bits.replayCarry 1381 s2_out.rep_info.mshr_id := io.dcache.resp.bits.mshr_id 1382 s2_out.rep_info.last_beat := s2_in.paddr(log2Up(refillBytes)) 1383 s2_out.rep_info.debug := s2_in.uop.debugInfo 1384 s2_out.rep_info.tlb_id := io.tlb_hint.id 1385 s2_out.rep_info.tlb_full := io.tlb_hint.full 1386 1387 // if forward fail, replay this inst from fetch 1388 val debug_fwd_fail_rep = s2_fwd_fail && !s2_troublem && !s2_in.tlbMiss 1389 // if ld-ld violation is detected, replay from this inst from fetch 1390 val debug_ldld_nuke_rep = false.B // s2_ldld_violation && !s2_mmio && !s2_is_prefetch && !s2_in.tlbMiss 1391 1392 // to be removed 1393 io.feedback_fast.valid := false.B 1394 io.feedback_fast.bits.hit := false.B 1395 io.feedback_fast.bits.flushState := s2_in.ptwBack 1396 io.feedback_fast.bits.robIdx := s2_in.uop.robIdx 1397 io.feedback_fast.bits.sqIdx := s2_in.uop.sqIdx 1398 io.feedback_fast.bits.lqIdx := s2_in.uop.lqIdx 1399 io.feedback_fast.bits.sourceType := RSFeedbackType.lrqFull 1400 io.feedback_fast.bits.dataInvalidSqIdx := DontCare 1401 1402 io.ldCancel.ld1Cancel := false.B 1403 1404 // fast wakeup 1405 val s1_fast_uop_valid = WireInit(false.B) 1406 s1_fast_uop_valid := 1407 !io.dcache.s1_disable_fast_wakeup && 1408 s1_valid && 1409 !s1_kill && 1410 !io.tlb.resp.bits.miss && 1411 !io.lsq.forward.dataInvalidFast 1412 io.fast_uop.valid := GatedValidRegNext(s1_fast_uop_valid) && (s2_valid && !s2_out.rep_info.need_rep && !s2_uncache && !(s2_prf && !s2_hw_prf)) && !s2_isvec && !s2_frm_mabuf 1413 io.fast_uop.bits := RegEnable(s1_out.uop, s1_fast_uop_valid) 1414 1415 // 1416 io.s2_ptr_chasing := RegEnable(s1_try_ptr_chasing && !s1_cancel_ptr_chasing, false.B, s1_fire) 1417 1418 // RegNext prefetch train for better timing 1419 // ** Now, prefetch train is valid at load s3 ** 1420 val s2_prefetch_train_valid = WireInit(false.B) 1421 s2_prefetch_train_valid := s2_valid && !s2_actually_uncache && (!s2_in.tlbMiss || s2_hw_prf) 1422 io.prefetch_train.valid := GatedValidRegNext(s2_prefetch_train_valid) 1423 io.prefetch_train.bits.fromLsPipelineBundle(s2_in, latch = true, enable = s2_prefetch_train_valid) 1424 io.prefetch_train.bits.miss := RegEnable(io.dcache.resp.bits.miss, s2_prefetch_train_valid) // TODO: use trace with bank conflict? 1425 io.prefetch_train.bits.meta_prefetch := RegEnable(io.dcache.resp.bits.meta_prefetch, s2_prefetch_train_valid) 1426 io.prefetch_train.bits.meta_access := RegEnable(io.dcache.resp.bits.meta_access, s2_prefetch_train_valid) 1427 io.prefetch_train.bits.isFinalSplit := false.B 1428 io.prefetch_train.bits.misalignWith16Byte := false.B 1429 io.prefetch_train.bits.misalignNeedWakeUp := false.B 1430 io.prefetch_train.bits.updateAddrValid := false.B 1431 io.prefetch_train.bits.isMisalign := false.B 1432 io.prefetch_train.bits.hasException := false.B 1433 io.s1_prefetch_spec := s1_fire 1434 io.s2_prefetch_spec := s2_prefetch_train_valid 1435 1436 val s2_prefetch_train_l1_valid = WireInit(false.B) 1437 s2_prefetch_train_l1_valid := s2_valid && !s2_actually_uncache 1438 io.prefetch_train_l1.valid := GatedValidRegNext(s2_prefetch_train_l1_valid) 1439 io.prefetch_train_l1.bits.fromLsPipelineBundle(s2_in, latch = true, enable = s2_prefetch_train_l1_valid) 1440 io.prefetch_train_l1.bits.miss := RegEnable(io.dcache.resp.bits.miss, s2_prefetch_train_l1_valid) 1441 io.prefetch_train_l1.bits.meta_prefetch := RegEnable(io.dcache.resp.bits.meta_prefetch, s2_prefetch_train_l1_valid) 1442 io.prefetch_train_l1.bits.meta_access := RegEnable(io.dcache.resp.bits.meta_access, s2_prefetch_train_l1_valid) 1443 io.prefetch_train_l1.bits.isFinalSplit := false.B 1444 io.prefetch_train_l1.bits.misalignWith16Byte := false.B 1445 io.prefetch_train_l1.bits.misalignNeedWakeUp := false.B 1446 io.prefetch_train_l1.bits.updateAddrValid := false.B 1447 io.prefetch_train_l1.bits.hasException := false.B 1448 io.prefetch_train_l1.bits.isMisalign := false.B 1449 if (env.FPGAPlatform){ 1450 io.dcache.s0_pc := DontCare 1451 io.dcache.s1_pc := DontCare 1452 io.dcache.s2_pc := DontCare 1453 }else{ 1454 io.dcache.s0_pc := s0_out.uop.pc 1455 io.dcache.s1_pc := s1_out.uop.pc 1456 io.dcache.s2_pc := s2_out.uop.pc 1457 } 1458 io.dcache.s2_kill := s2_pmp.ld || s2_actually_uncache || s2_kill 1459 1460 val s1_ld_left_fire = s1_valid && !s1_kill && s2_ready 1461 val s2_ld_valid_dup = RegInit(0.U(6.W)) 1462 s2_ld_valid_dup := 0x0.U(6.W) 1463 when (s1_ld_left_fire && !s1_out.isHWPrefetch) { s2_ld_valid_dup := 0x3f.U(6.W) } 1464 when (s1_kill || s1_out.isHWPrefetch) { s2_ld_valid_dup := 0x0.U(6.W) } 1465 assert(RegNext((s2_valid === s2_ld_valid_dup(0)) || RegNext(s1_out.isHWPrefetch))) 1466 1467 // Pipeline 1468 // -------------------------------------------------------------------------------- 1469 // stage 3 1470 // -------------------------------------------------------------------------------- 1471 // writeback and update load queue 1472 val s3_valid = GatedValidRegNext(s2_valid && !s2_out.isHWPrefetch && !s2_out.uop.robIdx.needFlush(io.redirect)) 1473 val s3_in = RegEnable(s2_out, s2_fire) 1474 val s3_out = Wire(Valid(new MemExuOutput)) 1475 val s3_dcache_rep = RegEnable(s2_dcache_fast_rep && s2_troublem, false.B, s2_fire) 1476 val s3_ld_valid_dup = RegEnable(s2_ld_valid_dup, s2_fire) 1477 val s3_fast_rep = Wire(Bool()) 1478 val s3_nc_with_data = RegNext(s2_nc_with_data) 1479 val s3_troublem = GatedValidRegNext(s2_troublem) 1480 val s3_kill = s3_in.uop.robIdx.needFlush(io.redirect) 1481 val s3_vecout = Wire(new OnlyVecExuOutput) 1482 val s3_vecActive = RegEnable(s2_out.vecActive, true.B, s2_fire) 1483 val s3_isvec = RegEnable(s2_out.isvec, false.B, s2_fire) 1484 val s3_vec_alignedType = RegEnable(s2_out.alignedType, s2_fire) 1485 val s3_vec_mBIndex = RegEnable(s2_out.mbIndex, s2_fire) 1486 val s3_frm_mabuf = s3_in.isFrmMisAlignBuf 1487 val s3_mmio_req = RegNext(s2_mmio_req) 1488 val s3_pdest = RegNext(Mux(s2_valid, s2_out.uop.pdest, s2_mmio_req.bits.uop.pdest)) 1489 val s3_rfWen = RegEnable(Mux(s2_valid, s2_out.uop.rfWen, s2_mmio_req.bits.uop.rfWen), s2_valid || s2_mmio_req.valid) 1490 val s3_fpWen = RegEnable(Mux(s2_valid, s2_out.uop.fpWen, s2_mmio_req.bits.uop.fpWen), s2_valid || s2_mmio_req.valid) 1491 val s3_data_select = RegEnable(s2_data_select, 0.U(s2_data_select.getWidth.W), s2_fire) 1492 val s3_data_select_by_offset = RegEnable(s2_data_select_by_offset, 0.U.asTypeOf(s2_data_select_by_offset), s2_fire) 1493 val s3_hw_err = 1494 if (EnableAccurateLoadError) { 1495 io.dcache.resp.bits.error_delayed && GatedValidRegNext(io.csrCtrl.cache_error_enable) && s3_troublem 1496 } else { 1497 WireInit(false.B) 1498 } 1499 val s3_safe_wakeup = RegEnable(s2_safe_wakeup, s2_fire) 1500 val s3_safe_writeback = RegEnable(s2_safe_writeback, s2_fire) || s3_hw_err 1501 val s3_exception = RegEnable(s2_real_exception, s2_fire) 1502 val s3_mis_align = RegEnable(s2_mis_align, s2_fire) 1503 val s3_trigger_debug_mode = RegEnable(s2_trigger_debug_mode, false.B, s2_fire) 1504 1505 // TODO: Fix vector load merge buffer nack 1506 val s3_vec_mb_nack = Wire(Bool()) 1507 s3_vec_mb_nack := false.B 1508 XSError(s3_valid && s3_vec_mb_nack, "Merge buffer should always accept vector loads!") 1509 1510 s3_ready := !s3_valid || s3_kill || io.ldout.ready 1511 1512 1513 // forwrad last beat 1514 val s3_fast_rep_canceled = io.replay.valid && io.replay.bits.forward_tlDchannel || io.misalign_ldin.valid || !io.dcache.req.ready 1515 1516 val s3_can_enter_lsq_valid = s3_valid && (!s3_fast_rep || s3_fast_rep_canceled) && !s3_in.feedbacked && !s3_nc_with_data && !s3_in.misalignNeedWakeUp 1517 io.lsq.ldin.valid := s3_can_enter_lsq_valid 1518 // TODO: check this --by hx 1519 // io.lsq.ldin.valid := s3_valid && (!s3_fast_rep || !io.fast_rep_out.ready) && !s3_in.feedbacked && !s3_in.lateKill 1520 io.lsq.ldin.bits := s3_in 1521 io.lsq.ldin.bits.miss := s3_in.miss 1522 1523 // connect to misalignBuffer 1524 val toMisalignBufferValid = s3_can_enter_lsq_valid && s3_mis_align && !s3_frm_mabuf 1525 io.misalign_buf.valid := toMisalignBufferValid 1526 io.misalign_buf.bits := s3_in 1527 1528 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1529 io.lsq.ldin.bits.data_wen_dup := s3_ld_valid_dup.asBools 1530 io.lsq.ldin.bits.replacementUpdated := io.dcache.resp.bits.replacementUpdated 1531 io.lsq.ldin.bits.missDbUpdated := GatedValidRegNext(s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated) 1532 io.lsq.ldin.bits.updateAddrValid := !s3_mis_align && (!s3_frm_mabuf || s3_in.isFinalSplit) || s3_exception 1533 io.lsq.ldin.bits.hasException := false.B 1534 1535 io.s3_dly_ld_err := false.B // s3_dly_ld_err && s3_valid 1536 io.lsq.ldin.bits.dcacheRequireReplay := s3_dcache_rep 1537 1538 val s3_vp_match_fail = GatedValidRegNext(s2_fwd_vp_match_invalid) && s3_troublem 1539 val s3_rep_frm_fetch = s3_vp_match_fail 1540 val s3_ldld_rep_inst = 1541 io.lsq.ldld_nuke_query.resp.valid && 1542 io.lsq.ldld_nuke_query.resp.bits.rep_frm_fetch && 1543 GatedValidRegNext(io.csrCtrl.ldld_vio_check_enable) 1544 val s3_flushPipe = s3_ldld_rep_inst 1545 1546 val s3_lrq_rep_info = WireInit(s3_in.rep_info) 1547 s3_lrq_rep_info.misalign_nack := toMisalignBufferValid && !io.misalign_buf.ready 1548 val s3_lrq_sel_rep_cause = PriorityEncoderOH(s3_lrq_rep_info.cause.asUInt) 1549 val s3_replayqueue_rep_cause = WireInit(0.U.asTypeOf(s3_in.rep_info.cause)) 1550 s3_replayqueue_rep_cause(LoadReplayCauses.C_MF) := s3_mis_align && s3_lrq_rep_info.misalign_nack 1551 1552 val s3_mab_rep_info = WireInit(s3_in.rep_info) 1553 val s3_mab_sel_rep_cause = PriorityEncoderOH(s3_mab_rep_info.cause.asUInt) 1554 val s3_misalign_rep_cause = WireInit(0.U.asTypeOf(s3_in.rep_info.cause)) 1555 1556 s3_misalign_rep_cause := Mux( 1557 s3_in.misalignNeedWakeUp, 1558 0.U.asTypeOf(s3_mab_rep_info.cause.cloneType), 1559 VecInit(s3_mab_sel_rep_cause.asBools) 1560 ) 1561 1562 when (s3_exception || s3_hw_err || s3_rep_frm_fetch || s3_frm_mabuf) { 1563 s3_replayqueue_rep_cause := 0.U.asTypeOf(s3_lrq_rep_info.cause.cloneType) 1564 } .otherwise { 1565 s3_replayqueue_rep_cause := VecInit(s3_lrq_sel_rep_cause.asBools) 1566 1567 } 1568 io.lsq.ldin.bits.rep_info.cause := s3_replayqueue_rep_cause 1569 1570 1571 // Int load, if hit, will be writebacked at s3 1572 s3_out.valid := s3_valid && s3_safe_writeback && !toMisalignBufferValid 1573 s3_out.bits.uop := s3_in.uop 1574 s3_out.bits.uop.fpWen := s3_in.uop.fpWen 1575 s3_out.bits.uop.exceptionVec(loadAccessFault) := s3_in.uop.exceptionVec(loadAccessFault) && s3_vecActive 1576 s3_out.bits.uop.exceptionVec(hardwareError) := s3_hw_err && s3_vecActive 1577 s3_out.bits.uop.flushPipe := false.B 1578 s3_out.bits.uop.replayInst := false.B 1579 s3_out.bits.data := s3_in.data 1580 s3_out.bits.isFromLoadUnit := true.B 1581 s3_out.bits.debug.isMMIO := s3_in.mmio 1582 s3_out.bits.debug.isNC := s3_in.nc 1583 s3_out.bits.debug.isPerfCnt := false.B 1584 s3_out.bits.debug.paddr := s3_in.paddr 1585 s3_out.bits.debug.vaddr := s3_in.vaddr 1586 1587 // Vector load, writeback to merge buffer 1588 // TODO: Add assertion in merge buffer, merge buffer must accept vec load writeback 1589 s3_vecout.isvec := s3_isvec 1590 s3_vecout.vecdata := 0.U // Data will be assigned later 1591 s3_vecout.mask := s3_in.mask 1592 // s3_vecout.rob_idx_valid := s3_in.rob_idx_valid 1593 // s3_vecout.inner_idx := s3_in.inner_idx 1594 // s3_vecout.rob_idx := s3_in.rob_idx 1595 // s3_vecout.offset := s3_in.offset 1596 s3_vecout.reg_offset := s3_in.reg_offset 1597 s3_vecout.vecActive := s3_vecActive 1598 s3_vecout.is_first_ele := s3_in.is_first_ele 1599 // s3_vecout.uopQueuePtr := DontCare // uopQueuePtr is already saved in flow queue 1600 // s3_vecout.flowPtr := s3_in.flowPtr 1601 s3_vecout.elemIdx := s3_in.elemIdx // elemIdx is already saved in flow queue // TODO: 1602 s3_vecout.elemIdxInsideVd := s3_in.elemIdxInsideVd 1603 s3_vecout.trigger := s3_in.uop.trigger 1604 s3_vecout.vstart := s3_in.uop.vpu.vstart 1605 s3_vecout.vecTriggerMask := s3_in.vecTriggerMask 1606 val s3_usSecondInv = s3_in.usSecondInv 1607 1608 val s3_frm_mis_flush = s3_frm_mabuf && 1609 (io.misalign_ldout.bits.rep_info.fwd_fail || io.misalign_ldout.bits.rep_info.mem_amb || io.misalign_ldout.bits.rep_info.nuke) 1610 1611 io.rollback.valid := s3_valid && (s3_rep_frm_fetch || s3_flushPipe || s3_frm_mis_flush) && !s3_exception 1612 io.rollback.bits := DontCare 1613 io.rollback.bits.isRVC := s3_out.bits.uop.preDecodeInfo.isRVC 1614 io.rollback.bits.robIdx := s3_out.bits.uop.robIdx 1615 io.rollback.bits.ftqIdx := s3_out.bits.uop.ftqPtr 1616 io.rollback.bits.ftqOffset := s3_out.bits.uop.ftqOffset 1617 io.rollback.bits.level := Mux(s3_rep_frm_fetch || s3_frm_mis_flush, RedirectLevel.flush, RedirectLevel.flushAfter) 1618 io.rollback.bits.cfiUpdate.target := s3_out.bits.uop.pc 1619 io.rollback.bits.debug_runahead_checkpoint_id := s3_out.bits.uop.debugInfo.runahead_checkpoint_id 1620 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1621 1622 io.lsq.ldin.bits.uop := s3_out.bits.uop 1623// io.lsq.ldin.bits.uop.exceptionVec(loadAddrMisaligned) := Mux(s3_in.onlyMisalignException, false.B, s3_in.uop.exceptionVec(loadAddrMisaligned)) 1624 1625 val s3_revoke = s3_exception || io.lsq.ldin.bits.rep_info.need_rep || s3_mis_align 1626 io.lsq.ldld_nuke_query.revoke := s3_revoke 1627 io.lsq.stld_nuke_query.revoke := s3_revoke 1628 1629 // feedback slow 1630 s3_fast_rep := RegNext(s2_fast_rep) 1631 1632 val s3_fb_no_waiting = !s3_in.isLoadReplay && 1633 (!(s3_fast_rep && !s3_fast_rep_canceled)) && 1634 !s3_in.feedbacked 1635 1636 // feedback: scalar load will send feedback to RS 1637 // vector load will send signal to VL Merge Buffer, then send feedback at granularity of uops 1638 io.feedback_slow.valid := s3_valid && s3_fb_no_waiting && !s3_isvec && !s3_frm_mabuf 1639 io.feedback_slow.bits.hit := !s3_lrq_rep_info.need_rep || io.lsq.ldin.ready 1640 io.feedback_slow.bits.flushState := s3_in.ptwBack 1641 io.feedback_slow.bits.robIdx := s3_in.uop.robIdx 1642 io.feedback_slow.bits.sqIdx := s3_in.uop.sqIdx 1643 io.feedback_slow.bits.lqIdx := s3_in.uop.lqIdx 1644 io.feedback_slow.bits.sourceType := RSFeedbackType.lrqFull 1645 io.feedback_slow.bits.dataInvalidSqIdx := DontCare 1646 1647 // TODO: vector wakeup? 1648 io.ldCancel.ld2Cancel := s3_valid && !s3_safe_wakeup && !s3_isvec && (!s3_frm_mabuf || s3_in.misalignNeedWakeUp) 1649 1650 val s3_ld_wb_meta = Mux(s3_valid, s3_out.bits, s3_mmio_req.bits) 1651 1652 // data from load queue refill 1653 val s3_ld_raw_data_frm_mmio = RegNextN(io.lsq.ld_raw_data, 3) 1654 val s3_merged_data_frm_mmio = s3_ld_raw_data_frm_mmio.mergedData() 1655 val s3_picked_data_frm_mmio = LookupTree(s3_ld_raw_data_frm_mmio.addrOffset, List( 1656 "b000".U -> s3_merged_data_frm_mmio(63, 0), 1657 "b001".U -> s3_merged_data_frm_mmio(63, 8), 1658 "b010".U -> s3_merged_data_frm_mmio(63, 16), 1659 "b011".U -> s3_merged_data_frm_mmio(63, 24), 1660 "b100".U -> s3_merged_data_frm_mmio(63, 32), 1661 "b101".U -> s3_merged_data_frm_mmio(63, 40), 1662 "b110".U -> s3_merged_data_frm_mmio(63, 48), 1663 "b111".U -> s3_merged_data_frm_mmio(63, 56) 1664 )) 1665 val s3_ld_data_frm_mmio = rdataHelper(s3_ld_raw_data_frm_mmio.uop, s3_picked_data_frm_mmio) 1666 1667 /* data from pipe, which forward from respectively 1668 * dcache hit: [D channel, mshr, sbuffer, sq] 1669 * nc_with_data: [sq] 1670 */ 1671 1672 val s2_ld_data_frm_nc = shiftDataToHigh(s2_out.paddr, s2_out.data) 1673 val s2_ld_raw_data_frm_pipe = Wire(new LoadDataFromDcacheBundle) 1674 s2_ld_raw_data_frm_pipe.respDcacheData := Mux(s2_nc_with_data, s2_ld_data_frm_nc, io.dcache.resp.bits.data) 1675 s2_ld_raw_data_frm_pipe.forward_D := s2_fwd_frm_d_chan && !s2_nc_with_data 1676 s2_ld_raw_data_frm_pipe.forwardData_D := s2_fwd_data_frm_d_chan 1677 s2_ld_raw_data_frm_pipe.forward_mshr := s2_fwd_frm_mshr && !s2_nc_with_data 1678 s2_ld_raw_data_frm_pipe.forwardData_mshr := s2_fwd_data_frm_mshr 1679 s2_ld_raw_data_frm_pipe.forward_result_valid := s2_fwd_data_valid 1680 1681 s2_ld_raw_data_frm_pipe.forwardMask := s2_fwd_mask 1682 s2_ld_raw_data_frm_pipe.forwardData := s2_fwd_data 1683 s2_ld_raw_data_frm_pipe.uop := s2_out.uop 1684 s2_ld_raw_data_frm_pipe.addrOffset := s2_out.paddr(3, 0) 1685 1686 val s2_ld_raw_data_frm_tlD = s2_ld_raw_data_frm_pipe.mergeTLData() 1687 val s2_merged_data_frm_pipe = s2_ld_raw_data_frm_pipe.mergeLsqFwdData(s2_ld_raw_data_frm_tlD) 1688 val s3_merged_data_frm_pipe = RegEnable(s2_merged_data_frm_pipe, s2_fire) 1689 1690 // duplicate reg for ldout and vecldout 1691 private val LdDataDup = 3 1692 require(LdDataDup >= 2) 1693 1694 val s3_data_frm_pipe = VecInit((0 until LdDataDup).map(i => { 1695 VecInit(Seq( 1696 s3_merged_data_frm_pipe(63, 0), 1697 s3_merged_data_frm_pipe(71, 8), 1698 s3_merged_data_frm_pipe(79, 16), 1699 s3_merged_data_frm_pipe(87, 24), 1700 s3_merged_data_frm_pipe(95, 32), 1701 s3_merged_data_frm_pipe(103, 40), 1702 s3_merged_data_frm_pipe(111, 48), 1703 s3_merged_data_frm_pipe(119, 56), 1704 s3_merged_data_frm_pipe(127, 64), 1705 s3_merged_data_frm_pipe(127, 72), 1706 s3_merged_data_frm_pipe(127, 80), 1707 s3_merged_data_frm_pipe(127, 88), 1708 s3_merged_data_frm_pipe(127, 96), 1709 s3_merged_data_frm_pipe(127, 104), 1710 s3_merged_data_frm_pipe(127, 112), 1711 s3_merged_data_frm_pipe(127, 120), 1712 )) 1713 })) 1714 val s3_picked_data_frm_pipe = VecInit((0 until LdDataDup).map(i => { 1715 Mux1H(s3_data_select_by_offset, s3_data_frm_pipe(i)) 1716 })) 1717 val s3_ld_data_frm_pipe = VecInit((0 until LdDataDup).map(i => { 1718 newRdataHelper(s3_data_select, s3_picked_data_frm_pipe(i)) 1719 })) 1720 1721 // FIXME: add 1 cycle delay ? 1722 // io.lsq.uncache.ready := !s3_valid 1723 val s3_ldout_valid = s3_mmio_req.valid || 1724 s3_out.valid && RegNext(!s2_out.isvec && !s2_out.isFrmMisAlignBuf) 1725 val s3_outexception = ExceptionNO.selectByFu(s3_out.bits.uop.exceptionVec, LduCfg).asUInt.orR && s3_vecActive 1726 io.ldout.valid := s3_ldout_valid 1727 io.ldout.bits := s3_ld_wb_meta 1728 io.ldout.bits.data := Mux(s3_valid, s3_ld_data_frm_pipe(0), s3_ld_data_frm_mmio) 1729 io.ldout.bits.uop.rfWen := s3_rfWen 1730 io.ldout.bits.uop.fpWen := s3_fpWen 1731 io.ldout.bits.uop.pdest := s3_pdest 1732 io.ldout.bits.uop.exceptionVec := ExceptionNO.selectByFu(s3_ld_wb_meta.uop.exceptionVec, LduCfg) 1733 io.ldout.bits.isFromLoadUnit := true.B 1734 io.ldout.bits.uop.fuType := Mux( 1735 s3_valid && s3_isvec, 1736 FuType.vldu.U, 1737 FuType.ldu.U 1738 ) 1739 1740 XSError(s3_valid && s3_in.misalignNeedWakeUp && !s3_frm_mabuf, "Only the needwakeup from the misalignbuffer may be high") 1741 XSError(s3_valid && s3_vecout.isvec && s3_in.vecActive && !s3_vecout.mask.orR, "In vecActive, mask complement should not be 0") 1742 // TODO: check this --hx 1743 // io.ldout.valid := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) && !s3_vecout.isvec || 1744 // io.lsq.uncache.valid && !io.lsq.uncache.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid && !io.lsq.uncache.bits.isVls 1745 // io.ldout.bits.data := Mux(s3_out.valid, s3_ld_data_frm_pipe, s3_ld_data_frm_mmio) 1746 // io.ldout.valid := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) || 1747 // s3_mmio_req.valid && !s3_mmio_req.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid 1748 1749 // s3 load fast replay 1750 io.fast_rep_out.valid := s3_valid && s3_fast_rep 1751 io.fast_rep_out.bits := s3_in 1752 io.fast_rep_out.bits.lateKill := s3_rep_frm_fetch 1753 io.fast_rep_out.bits.delayedLoadError := s3_hw_err 1754 1755 val vecFeedback = s3_valid && s3_fb_no_waiting && s3_lrq_rep_info.need_rep && !io.lsq.ldin.ready && s3_isvec 1756 1757 // vector output 1758 io.vecldout.bits.alignedType := s3_vec_alignedType 1759 // vec feedback 1760 io.vecldout.bits.vecFeedback := vecFeedback 1761 // TODO: VLSU, uncache data logic 1762 val vecdata = rdataVecHelper(s3_vec_alignedType(1,0), s3_picked_data_frm_pipe(1)) 1763 io.vecldout.bits.vecdata.get := Mux( 1764 s3_in.misalignWith16Byte, 1765 s3_picked_data_frm_pipe(1), 1766 Mux( 1767 s3_in.is128bit, 1768 s3_merged_data_frm_pipe, 1769 vecdata 1770 ) 1771 ) 1772 io.vecldout.bits.isvec := s3_vecout.isvec 1773 io.vecldout.bits.elemIdx := s3_vecout.elemIdx 1774 io.vecldout.bits.elemIdxInsideVd.get := s3_vecout.elemIdxInsideVd 1775 io.vecldout.bits.mask := s3_vecout.mask 1776 io.vecldout.bits.hasException := s3_exception 1777 io.vecldout.bits.reg_offset.get := s3_vecout.reg_offset 1778 io.vecldout.bits.usSecondInv := s3_usSecondInv 1779 io.vecldout.bits.mBIndex := s3_vec_mBIndex 1780 io.vecldout.bits.hit := !s3_lrq_rep_info.need_rep || io.lsq.ldin.ready 1781 io.vecldout.bits.sourceType := RSFeedbackType.lrqFull 1782 io.vecldout.bits.trigger := s3_vecout.trigger 1783 io.vecldout.bits.flushState := DontCare 1784 io.vecldout.bits.exceptionVec := ExceptionNO.selectByFu(s3_out.bits.uop.exceptionVec, VlduCfg) 1785 io.vecldout.bits.vaddr := s3_in.fullva 1786 io.vecldout.bits.vaNeedExt := s3_in.vaNeedExt 1787 io.vecldout.bits.gpaddr := s3_in.gpaddr 1788 io.vecldout.bits.isForVSnonLeafPTE := s3_in.isForVSnonLeafPTE 1789 io.vecldout.bits.mmio := DontCare 1790 io.vecldout.bits.vstart := s3_vecout.vstart 1791 io.vecldout.bits.vecTriggerMask := s3_vecout.vecTriggerMask 1792 io.vecldout.bits.nc := DontCare 1793 1794 io.vecldout.valid := s3_out.valid && !s3_out.bits.uop.robIdx.needFlush(io.redirect) && s3_vecout.isvec && !s3_mis_align && !s3_frm_mabuf //|| 1795 // TODO: check this, why !io.lsq.uncache.bits.isVls before? 1796 // Now vector instruction don't support mmio. 1797 // io.lsq.uncache.valid && !io.lsq.uncache.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid && io.lsq.uncache.bits.isVls 1798 //io.lsq.uncache.valid && !io.lsq.uncache.bits.uop.robIdx.needFlush(io.redirect) && !s3_out.valid && !io.lsq.uncache.bits.isVls 1799 1800 io.misalign_ldout.valid := s3_valid && (!s3_fast_rep || s3_fast_rep_canceled) && s3_frm_mabuf 1801 io.misalign_ldout.bits := io.lsq.ldin.bits 1802 io.misalign_ldout.bits.data := Mux(s3_in.misalignWith16Byte, s3_merged_data_frm_pipe, s3_picked_data_frm_pipe(2)) 1803 io.misalign_ldout.bits.rep_info.cause := s3_misalign_rep_cause 1804 1805 // fast load to load forward 1806 if (EnableLoadToLoadForward) { 1807 io.l2l_fwd_out.valid := s3_valid && !s3_in.mmio && !s3_in.nc && !s3_lrq_rep_info.need_rep 1808 io.l2l_fwd_out.data := Mux(s3_in.vaddr(3), s3_merged_data_frm_pipe(127, 64), s3_merged_data_frm_pipe(63, 0)) 1809 io.l2l_fwd_out.dly_ld_err := s3_hw_err || // ecc delayed error 1810 s3_ldld_rep_inst || 1811 s3_rep_frm_fetch 1812 } else { 1813 io.l2l_fwd_out.valid := false.B 1814 io.l2l_fwd_out.data := DontCare 1815 io.l2l_fwd_out.dly_ld_err := DontCare 1816 } 1817 1818 // s1 1819 io.debug_ls.s1_robIdx := s1_in.uop.robIdx.value 1820 io.debug_ls.s1_isLoadToLoadForward := s1_fire && s1_try_ptr_chasing && !s1_ptr_chasing_canceled 1821 io.debug_ls.s1_isTlbFirstMiss := s1_fire && s1_tlb_miss && s1_in.isFirstIssue 1822 // s2 1823 io.debug_ls.s2_robIdx := s2_in.uop.robIdx.value 1824 io.debug_ls.s2_isBankConflict := s2_fire && (!s2_kill && s2_bank_conflict) 1825 io.debug_ls.s2_isDcacheFirstMiss := s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue 1826 io.debug_ls.s2_isForwardFail := s2_fire && s2_fwd_fail 1827 // s3 1828 io.debug_ls.s3_robIdx := s3_in.uop.robIdx.value 1829 io.debug_ls.s3_isReplayFast := s3_valid && s3_fast_rep && !s3_fast_rep_canceled 1830 io.debug_ls.s3_isReplayRS := RegNext(io.feedback_fast.valid && !io.feedback_fast.bits.hit) || (io.feedback_slow.valid && !io.feedback_slow.bits.hit) 1831 io.debug_ls.s3_isReplaySlow := io.lsq.ldin.valid && io.lsq.ldin.bits.rep_info.need_rep 1832 io.debug_ls.s3_isReplay := s3_valid && s3_lrq_rep_info.need_rep // include fast+slow+rs replay 1833 io.debug_ls.replayCause := s3_lrq_rep_info.cause 1834 io.debug_ls.replayCnt := 1.U 1835 1836 // Topdown 1837 io.lsTopdownInfo.s1.robIdx := s1_in.uop.robIdx.value 1838 io.lsTopdownInfo.s1.vaddr_valid := s1_valid && s1_in.hasROBEntry 1839 io.lsTopdownInfo.s1.vaddr_bits := s1_vaddr 1840 io.lsTopdownInfo.s2.robIdx := s2_in.uop.robIdx.value 1841 io.lsTopdownInfo.s2.paddr_valid := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss 1842 io.lsTopdownInfo.s2.paddr_bits := s2_in.paddr 1843 io.lsTopdownInfo.s2.first_real_miss := io.dcache.resp.bits.real_miss 1844 io.lsTopdownInfo.s2.cache_miss_en := s2_fire && s2_in.hasROBEntry && !s2_in.tlbMiss && !s2_in.missDbUpdated 1845 1846 // perf cnt 1847 XSPerfAccumulate("s0_in_valid", io.ldin.valid) 1848 XSPerfAccumulate("s0_in_block", io.ldin.valid && !io.ldin.fire) 1849 XSPerfAccumulate("s0_vecin_valid", io.vecldin.valid) 1850 XSPerfAccumulate("s0_vecin_block", io.vecldin.valid && !io.vecldin.fire) 1851 XSPerfAccumulate("s0_in_fire_first_issue", s0_valid && s0_sel_src.isFirstIssue) 1852 XSPerfAccumulate("s0_lsq_replay_issue", io.replay.fire) 1853 XSPerfAccumulate("s0_lsq_replay_vecissue", io.replay.fire && io.replay.bits.isvec) 1854 XSPerfAccumulate("s0_ldu_fire_first_issue", io.ldin.fire && s0_sel_src.isFirstIssue) 1855 XSPerfAccumulate("s0_fast_replay_issue", io.fast_rep_in.fire) 1856 XSPerfAccumulate("s0_fast_replay_vecissue", io.fast_rep_in.fire && io.fast_rep_in.bits.isvec) 1857 XSPerfAccumulate("s0_stall_out", s0_valid && !s0_can_go) 1858 XSPerfAccumulate("s0_stall_dcache", s0_valid && !io.dcache.req.ready) 1859 XSPerfAccumulate("s0_addr_spec_success", s0_fire && s0_dcache_vaddr(VAddrBits-1, 12) === io.ldin.bits.src(0)(VAddrBits-1, 12)) 1860 XSPerfAccumulate("s0_addr_spec_failed", s0_fire && s0_dcache_vaddr(VAddrBits-1, 12) =/= io.ldin.bits.src(0)(VAddrBits-1, 12)) 1861 XSPerfAccumulate("s0_addr_spec_success_once", s0_fire && s0_dcache_vaddr(VAddrBits-1, 12) === io.ldin.bits.src(0)(VAddrBits-1, 12) && s0_sel_src.isFirstIssue) 1862 XSPerfAccumulate("s0_addr_spec_failed_once", s0_fire && s0_dcache_vaddr(VAddrBits-1, 12) =/= io.ldin.bits.src(0)(VAddrBits-1, 12) && s0_sel_src.isFirstIssue) 1863 XSPerfAccumulate("s0_vec_addr_vlen_aligned", s0_fire && s0_sel_src.isvec && s0_dcache_vaddr(3, 0) === 0.U) 1864 XSPerfAccumulate("s0_vec_addr_vlen_unaligned", s0_fire && s0_sel_src.isvec && s0_dcache_vaddr(3, 0) =/= 0.U) 1865 XSPerfAccumulate("s0_forward_tl_d_channel", s0_out.forward_tlDchannel) 1866 XSPerfAccumulate("s0_hardware_prefetch_fire", s0_fire && s0_hw_prf_select) 1867 XSPerfAccumulate("s0_software_prefetch_fire", s0_fire && s0_sel_src.prf && s0_src_select_vec(int_iss_idx)) 1868 XSPerfAccumulate("s0_hardware_prefetch_blocked", io.prefetch_req.valid && !s0_hw_prf_select) 1869 XSPerfAccumulate("s0_hardware_prefetch_total", io.prefetch_req.valid) 1870 1871 XSPerfAccumulate("s3_rollback_total", io.rollback.valid) 1872 XSPerfAccumulate("s3_rep_frm_fetch_rollback", io.rollback.valid && s3_rep_frm_fetch) 1873 XSPerfAccumulate("s3_flushPipe_rollback", io.rollback.valid && s3_flushPipe) 1874 XSPerfAccumulate("s3_frm_mis_flush_rollback", io.rollback.valid && s3_frm_mis_flush) 1875 1876 XSPerfAccumulate("s1_in_valid", s1_valid) 1877 XSPerfAccumulate("s1_in_fire", s1_fire) 1878 XSPerfAccumulate("s1_in_fire_first_issue", s1_fire && s1_in.isFirstIssue) 1879 XSPerfAccumulate("s1_tlb_miss", s1_fire && s1_tlb_miss) 1880 XSPerfAccumulate("s1_tlb_miss_first_issue", s1_fire && s1_tlb_miss && s1_in.isFirstIssue) 1881 XSPerfAccumulate("s1_stall_out", s1_valid && !s1_can_go) 1882 XSPerfAccumulate("s1_dly_err", s1_valid && s1_fast_rep_dly_err) 1883 1884 XSPerfAccumulate("s2_in_valid", s2_valid) 1885 XSPerfAccumulate("s2_in_fire", s2_fire) 1886 XSPerfAccumulate("s2_in_fire_first_issue", s2_fire && s2_in.isFirstIssue) 1887 XSPerfAccumulate("s2_dcache_miss", s2_fire && io.dcache.resp.bits.miss) 1888 XSPerfAccumulate("s2_dcache_miss_first_issue", s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1889 XSPerfAccumulate("s2_dcache_real_miss_first_issue", s2_fire && io.dcache.resp.bits.miss && s2_in.isFirstIssue) 1890 XSPerfAccumulate("s2_full_forward", s2_fire && s2_full_fwd) 1891 XSPerfAccumulate("s2_dcache_miss_full_forward", s2_fire && s2_dcache_miss) 1892 XSPerfAccumulate("s2_fwd_frm_d_can", s2_valid && s2_fwd_frm_d_chan) 1893 XSPerfAccumulate("s2_fwd_frm_d_chan_or_mshr", s2_valid && s2_fwd_frm_d_chan_or_mshr) 1894 XSPerfAccumulate("s2_stall_out", s2_fire && !s2_can_go) 1895 XSPerfAccumulate("s2_prefetch", s2_fire && s2_prf) 1896 XSPerfAccumulate("s2_prefetch_ignored", s2_fire && s2_prf && io.dcache.s2_mq_nack) // ignore prefetch for mshr full / miss req port conflict 1897 XSPerfAccumulate("s2_prefetch_miss", s2_fire && s2_prf && io.dcache.resp.bits.miss) // prefetch req miss in l1 1898 XSPerfAccumulate("s2_prefetch_hit", s2_fire && s2_prf && !io.dcache.resp.bits.miss) // prefetch req hit in l1 1899 XSPerfAccumulate("s2_prefetch_accept", s2_fire && s2_prf && io.dcache.resp.bits.miss && !io.dcache.s2_mq_nack) // prefetch a missed line in l1, and l1 accepted it 1900 XSPerfAccumulate("s2_forward_req", s2_fire && s2_in.forward_tlDchannel) 1901 XSPerfAccumulate("s2_successfully_forward_channel_D", s2_fire && s2_fwd_frm_d_chan && s2_fwd_data_valid) 1902 XSPerfAccumulate("s2_successfully_forward_mshr", s2_fire && s2_fwd_frm_mshr && s2_fwd_data_valid) 1903 1904 XSPerfAccumulate("load_to_load_forward", s1_try_ptr_chasing && !s1_ptr_chasing_canceled) 1905 XSPerfAccumulate("load_to_load_forward_try", s1_try_ptr_chasing) 1906 XSPerfAccumulate("load_to_load_forward_fail", s1_cancel_ptr_chasing) 1907 XSPerfAccumulate("load_to_load_forward_fail_cancelled", s1_cancel_ptr_chasing && s1_ptr_chasing_canceled) 1908 XSPerfAccumulate("load_to_load_forward_fail_wakeup_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && s1_not_fast_match) 1909 XSPerfAccumulate("load_to_load_forward_fail_op_not_ld", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && s1_fu_op_type_not_ld) 1910 XSPerfAccumulate("load_to_load_forward_fail_addr_align", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && s1_addr_misaligned) 1911 XSPerfAccumulate("load_to_load_forward_fail_set_mismatch", s1_cancel_ptr_chasing && !s1_ptr_chasing_canceled && !s1_not_fast_match && !s1_fu_op_type_not_ld && !s1_addr_misaligned && s1_addr_mismatch) 1912 1913 XSPerfAccumulate("nc_ld_writeback", io.ldout.valid && s3_nc_with_data) 1914 XSPerfAccumulate("nc_ld_exception", s3_valid && s3_nc_with_data && s3_in.uop.exceptionVec.reduce(_ || _)) 1915 XSPerfAccumulate("nc_ldld_vio", s3_valid && s3_nc_with_data && s3_ldld_rep_inst) 1916 XSPerfAccumulate("nc_stld_vio", s3_valid && s3_nc_with_data && s3_in.rep_info.nuke) 1917 XSPerfAccumulate("nc_ldld_vioNack", s3_valid && s3_nc_with_data && s3_in.rep_info.rar_nack) 1918 XSPerfAccumulate("nc_stld_vioNack", s3_valid && s3_nc_with_data && s3_in.rep_info.raw_nack) 1919 XSPerfAccumulate("nc_stld_fwd", s3_valid && s3_nc_with_data && RegNext(s2_full_fwd)) 1920 XSPerfAccumulate("nc_stld_fwdNotReady", s3_valid && s3_nc_with_data && RegNext(s2_mem_amb || s2_fwd_fail)) 1921 XSPerfAccumulate("nc_stld_fwdAddrMismatch", s3_valid && s3_nc_with_data && s3_vp_match_fail) 1922 1923 // bug lyq: some signals in perfEvents are no longer suitable for the current MemBlock design 1924 // hardware performance counter 1925 val perfEvents = Seq( 1926 ("load_s0_in_fire ", s0_fire ), 1927 ("load_to_load_forward ", s1_fire && s1_try_ptr_chasing && !s1_ptr_chasing_canceled ), 1928 ("stall_dcache ", s0_valid && s0_can_go && !io.dcache.req.ready ), 1929 ("load_s1_in_fire ", s0_fire ), 1930 ("load_s1_tlb_miss ", s1_fire && io.tlb.resp.bits.miss ), 1931 ("load_s2_in_fire ", s1_fire ), 1932 ("load_s2_dcache_miss ", s2_fire && io.dcache.resp.bits.miss ), 1933 ) 1934 generatePerfEvent() 1935 1936 if (backendParams.debugEn){ 1937 dontTouch(s0_src_valid_vec) 1938 dontTouch(s0_src_ready_vec) 1939 dontTouch(s0_src_select_vec) 1940 dontTouch(s3_ld_data_frm_pipe) 1941 s3_data_select_by_offset.map(x=> dontTouch(x)) 1942 s3_data_frm_pipe.map(x=> dontTouch(x)) 1943 s3_picked_data_frm_pipe.map(x=> dontTouch(x)) 1944 } 1945 1946 XSDebug(io.ldout.fire, "ldout %x\n", io.ldout.bits.uop.pc) 1947 // end 1948} 1949