1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.fu.PMPRespBundle 27import xiangshan.backend.fu.FuConfig.LduCfg 28import xiangshan.backend.rob.{DebugLsInfoBundle, RobPtr} 29import xiangshan.backend.Bundles.{DynInst, MemExuInput, MemExuOutput} 30import xiangshan.cache._ 31import xiangshan.cache.dcache.ReplayCarry 32import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp} 33import xiangshan.mem.mdp._ 34 35class LoadToLsqReplayIO(implicit p: Parameters) extends XSBundle with HasDCacheParameters { 36 // mshr refill index 37 val missMSHRId = UInt(log2Up(cfg.nMissEntries).W) 38 // get full data from store queue and sbuffer 39 val canForwardFullData = Bool() 40 // wait for data from store inst's store queue index 41 val dataInvalidSqIdx = new SqPtr 42 // wait for address from store queue index 43 val addrInvalidSqIdx = new SqPtr 44 // replay carry 45 val replayCarry = new ReplayCarry 46 // data in last beat 47 val dataInLastBeat = Bool() 48 // replay cause 49 val cause = Vec(LoadReplayCauses.allCauses, Bool()) 50 // 51 // performance debug information 52 val debug = new PerfDebugInfo 53 54 // 55 def tlbMiss = cause(LoadReplayCauses.tlbMiss) 56 def waitStore = cause(LoadReplayCauses.waitStore) 57 def schedError = cause(LoadReplayCauses.schedError) 58 def rejectEnq = cause(LoadReplayCauses.rejectEnq) 59 def dcacheMiss = cause(LoadReplayCauses.dcacheMiss) 60 def bankConflict = cause(LoadReplayCauses.bankConflict) 61 def dcacheReplay = cause(LoadReplayCauses.dcacheReplay) 62 def forwardFail = cause(LoadReplayCauses.forwardFail) 63 64 def forceReplay() = rejectEnq || schedError || waitStore || tlbMiss 65 def needReplay() = cause.asUInt.orR 66} 67 68class LoadToReplayIO(implicit p: Parameters) extends XSBundle { 69 val req = ValidIO(new LqWriteBundle) 70 val resp = Input(UInt(log2Up(LoadQueueReplaySize).W)) 71} 72 73class LoadToLsqIO(implicit p: Parameters) extends XSBundle { 74 val loadIn = DecoupledIO(new LqWriteBundle) 75 val loadOut = Flipped(DecoupledIO(new MemExuOutput)) 76 val ldRawData = Input(new LoadDataFromLQBundle) 77 val forward = new PipeLoadForwardQueryIO 78 val storeLoadViolationQuery = new LoadViolationQueryIO 79 val loadLoadViolationQuery = new LoadViolationQueryIO 80 val trigger = Flipped(new LqTriggerIO) 81} 82 83class LoadToLoadIO(implicit p: Parameters) extends XSBundle { 84 // load to load fast path is limited to ld (64 bit) used as vaddr src1 only 85 val data = UInt(XLEN.W) 86 val valid = Bool() 87} 88 89class LoadUnitTriggerIO(implicit p: Parameters) extends XSBundle { 90 val tdata2 = Input(UInt(64.W)) 91 val matchType = Input(UInt(2.W)) 92 val tEnable = Input(Bool()) // timing is calculated before this 93 val addrHit = Output(Bool()) 94 val lastDataHit = Output(Bool()) 95} 96 97// Load Pipeline Stage 0 98// Generate addr, use addr to query DCache and DTLB 99class LoadUnit_S0(implicit p: Parameters) extends XSModule with HasDCacheParameters with HasCircularQueuePtrHelper { 100 val io = IO(new Bundle() { 101 val in = Flipped(Decoupled(new MemExuInput)) 102 val out = Decoupled(new LqWriteBundle) 103 val prefetch_in = Flipped(ValidIO(new L1PrefetchReq)) 104 val dtlbReq = DecoupledIO(new TlbReq) 105 val dcacheReq = DecoupledIO(new DCacheWordReq) 106 val fastpath = Input(new LoadToLoadIO) 107 val s0_kill = Input(Bool()) 108 // wire from lq to load pipeline 109 val replay = Flipped(Decoupled(new LsPipelineBundle)) 110 val s0_sqIdx = Output(new SqPtr) 111 // l2l 112 val l2lForward_select = Output(Bool()) 113 }) 114 require(LoadPipelineWidth == backendParams.LduCnt) 115 116 val s0_vaddr = Wire(UInt(VAddrBits.W)) 117 val s0_mask = Wire(UInt(8.W)) 118 val s0_uop = Wire(new DynInst) 119 val s0_isFirstIssue = Wire(Bool()) 120 val s0_sqIdx = Wire(new SqPtr) 121 val s0_tryFastpath = WireInit(false.B) 122 val s0_replayCarry = Wire(new ReplayCarry) // way info for way predict related logic 123 124 // default value 125 s0_replayCarry.valid := false.B 126 s0_replayCarry.real_way_en := 0.U 127 io.s0_sqIdx := s0_sqIdx 128 129 val s0_replayShouldWait = io.in.valid && isAfter(io.replay.bits.uop.robIdx, io.in.bits.uop.robIdx) 130 // load flow select/gen 131 // 132 // src0: load replayed by LSQ (io.replay) 133 // src1: hardware prefetch from prefetchor (high confidence) (io.prefetch) 134 // src2: int read / software prefetch first issue from RS (io.in) 135 // src3: vec read first issue from RS (TODO) 136 // src4: load try pointchaising when no issued or replayed load (io.fastpath) 137 // src5: hardware prefetch from prefetchor (high confidence) (io.prefetch) 138 139 // load flow source valid 140 val lfsrc0_loadReplay_valid = io.replay.valid && !s0_replayShouldWait 141 val lfsrc1_highconfhwPrefetch_valid = io.prefetch_in.valid && io.prefetch_in.bits.confidence > 0.U 142 val lfsrc2_intloadFirstIssue_valid = io.in.valid // int flow first issue or software prefetch 143 val lfsrc3_vecloadFirstIssue_valid = WireInit(false.B) // TODO 144 val lfsrc4_l2lForward_valid = io.fastpath.valid 145 val lfsrc5_lowconfhwPrefetch_valid = io.prefetch_in.valid && io.prefetch_in.bits.confidence === 0.U 146 dontTouch(lfsrc0_loadReplay_valid) 147 dontTouch(lfsrc1_highconfhwPrefetch_valid) 148 dontTouch(lfsrc2_intloadFirstIssue_valid) 149 dontTouch(lfsrc3_vecloadFirstIssue_valid) 150 dontTouch(lfsrc4_l2lForward_valid) 151 dontTouch(lfsrc5_lowconfhwPrefetch_valid) 152 153 // load flow source ready 154 val lfsrc_loadReplay_ready = WireInit(true.B) 155 val lfsrc_highconfhwPrefetch_ready = !lfsrc0_loadReplay_valid 156 val lfsrc_intloadFirstIssue_ready = !lfsrc0_loadReplay_valid && 157 !lfsrc1_highconfhwPrefetch_valid 158 val lfsrc_vecloadFirstIssue_ready = !lfsrc0_loadReplay_valid && 159 !lfsrc1_highconfhwPrefetch_valid && 160 !lfsrc2_intloadFirstIssue_valid 161 val lfsrc_l2lForward_ready = !lfsrc0_loadReplay_valid && 162 !lfsrc1_highconfhwPrefetch_valid && 163 !lfsrc2_intloadFirstIssue_valid && 164 !lfsrc3_vecloadFirstIssue_valid 165 val lfsrc_lowconfhwPrefetch_ready = !lfsrc0_loadReplay_valid && 166 !lfsrc1_highconfhwPrefetch_valid && 167 !lfsrc2_intloadFirstIssue_valid && 168 !lfsrc3_vecloadFirstIssue_valid && 169 !lfsrc4_l2lForward_valid 170 dontTouch(lfsrc_loadReplay_ready) 171 dontTouch(lfsrc_highconfhwPrefetch_ready) 172 dontTouch(lfsrc_intloadFirstIssue_ready) 173 dontTouch(lfsrc_vecloadFirstIssue_ready) 174 dontTouch(lfsrc_l2lForward_ready) 175 dontTouch(lfsrc_lowconfhwPrefetch_ready) 176 177 // load flow source select (OH) 178 val lfsrc_loadReplay_select = lfsrc0_loadReplay_valid && lfsrc_loadReplay_ready 179 val lfsrc_hwprefetch_select = lfsrc_highconfhwPrefetch_ready && lfsrc1_highconfhwPrefetch_valid || 180 lfsrc_lowconfhwPrefetch_ready && lfsrc5_lowconfhwPrefetch_valid 181 val lfsrc_intloadFirstIssue_select = lfsrc_intloadFirstIssue_ready && lfsrc2_intloadFirstIssue_valid 182 val lfsrc_vecloadFirstIssue_select = lfsrc_vecloadFirstIssue_ready && lfsrc3_vecloadFirstIssue_valid 183 val lfsrc_l2lForward_select = lfsrc_l2lForward_ready && lfsrc4_l2lForward_valid 184 assert(!lfsrc_vecloadFirstIssue_select) // to be added 185 dontTouch(lfsrc_loadReplay_select) 186 dontTouch(lfsrc_hwprefetch_select) 187 dontTouch(lfsrc_intloadFirstIssue_select) 188 dontTouch(lfsrc_vecloadFirstIssue_select) 189 dontTouch(lfsrc_l2lForward_select) 190 191 io.l2lForward_select := lfsrc_l2lForward_select 192 193 // s0_valid == ture iff there is a valid load flow in load_s0 194 val s0_valid = lfsrc0_loadReplay_valid || 195 lfsrc1_highconfhwPrefetch_valid || 196 lfsrc2_intloadFirstIssue_valid || 197 lfsrc3_vecloadFirstIssue_valid || 198 lfsrc4_l2lForward_valid || 199 lfsrc5_lowconfhwPrefetch_valid 200 201 // prefetch related ctrl signal 202 val isPrefetch = WireInit(false.B) 203 val isPrefetchRead = WireInit(s0_uop.fuOpType === LSUOpType.prefetch_r) 204 val isPrefetchWrite = WireInit(s0_uop.fuOpType === LSUOpType.prefetch_w) 205 val isHWPrefetch = lfsrc_hwprefetch_select 206 207 // query DTLB 208 io.dtlbReq.valid := s0_valid 209 // hw prefetch addr does not need to be translated, give tlb paddr 210 io.dtlbReq.bits.vaddr := Mux(lfsrc_hwprefetch_select, io.prefetch_in.bits.paddr, s0_vaddr) 211 io.dtlbReq.bits.cmd := Mux(isPrefetch, 212 Mux(isPrefetchWrite, TlbCmd.write, TlbCmd.read), 213 TlbCmd.read 214 ) 215 io.dtlbReq.bits.size := LSUOpType.size(s0_uop.fuOpType) 216 io.dtlbReq.bits.kill := DontCare 217 io.dtlbReq.bits.memidx.is_ld := true.B 218 io.dtlbReq.bits.memidx.is_st := false.B 219 io.dtlbReq.bits.memidx.idx := s0_uop.lqIdx.value 220 io.dtlbReq.bits.debug.robIdx := s0_uop.robIdx 221 // hw prefetch addr does not need to be translated 222 io.dtlbReq.bits.no_translate := lfsrc_hwprefetch_select 223 io.dtlbReq.bits.debug.pc := s0_uop.pc 224 io.dtlbReq.bits.debug.isFirstIssue := s0_isFirstIssue 225 226 // query DCache 227 io.dcacheReq.valid := s0_valid 228 when (isPrefetchRead) { 229 io.dcacheReq.bits.cmd := MemoryOpConstants.M_PFR 230 }.elsewhen (isPrefetchWrite) { 231 io.dcacheReq.bits.cmd := MemoryOpConstants.M_PFW 232 }.otherwise { 233 io.dcacheReq.bits.cmd := MemoryOpConstants.M_XRD 234 } 235 io.dcacheReq.bits.addr := s0_vaddr 236 io.dcacheReq.bits.mask := s0_mask 237 io.dcacheReq.bits.data := DontCare 238 io.dcacheReq.bits.isFirstIssue := s0_isFirstIssue 239 when(isPrefetch) { 240 io.dcacheReq.bits.instrtype := DCACHE_PREFETCH_SOURCE.U 241 }.otherwise { 242 io.dcacheReq.bits.instrtype := LOAD_SOURCE.U 243 } 244 io.dcacheReq.bits.debug_robIdx := s0_uop.robIdx.value 245 io.dcacheReq.bits.replayCarry := s0_replayCarry 246 247 // TODO: update cache meta 248 io.dcacheReq.bits.id := DontCare 249 250 // assign default value 251 s0_uop := DontCare 252 // load flow priority mux 253 when(lfsrc_loadReplay_select) { 254 s0_vaddr := io.replay.bits.vaddr 255 s0_mask := genWmask(io.replay.bits.vaddr, io.replay.bits.uop.fuOpType(1, 0)) 256 s0_uop := io.replay.bits.uop 257 s0_isFirstIssue := io.replay.bits.isFirstIssue 258 s0_sqIdx := io.replay.bits.uop.sqIdx 259 s0_replayCarry := io.replay.bits.replayCarry 260 val replayUopIsPrefetch = WireInit(LSUOpType.isPrefetch(io.replay.bits.uop.fuOpType)) 261 when (replayUopIsPrefetch) { 262 isPrefetch := true.B 263 } 264 }.elsewhen(lfsrc_hwprefetch_select) { 265 // vaddr based index for dcache 266 s0_vaddr := io.prefetch_in.bits.getVaddr() 267 s0_mask := 0.U 268 s0_uop := DontCare 269 s0_isFirstIssue := false.B 270 s0_sqIdx := DontCare 271 s0_replayCarry := DontCare 272 // ctrl signal 273 isPrefetch := true.B 274 isPrefetchRead := !io.prefetch_in.bits.is_store 275 isPrefetchWrite := io.prefetch_in.bits.is_store 276 }.elsewhen(lfsrc_intloadFirstIssue_select) { 277 val imm12 = io.in.bits.uop.imm(11, 0) 278 s0_vaddr := io.in.bits.src(0) + SignExt(imm12, VAddrBits) 279 s0_mask := genWmask(s0_vaddr, io.in.bits.uop.fuOpType(1,0)) 280 s0_uop := io.in.bits.uop 281 s0_isFirstIssue := true.B 282 s0_sqIdx := io.in.bits.uop.sqIdx 283 val issueUopIsPrefetch = WireInit(LSUOpType.isPrefetch(io.in.bits.uop.fuOpType)) 284 when (issueUopIsPrefetch) { 285 isPrefetch := true.B 286 } 287 }.otherwise { 288 if (EnableLoadToLoadForward) { 289 s0_tryFastpath := lfsrc_l2lForward_select 290 // When there's no valid instruction from RS and LSQ, we try the load-to-load forwarding. 291 s0_vaddr := io.fastpath.data 292 // Assume the pointer chasing is always ld. 293 s0_uop.fuOpType := LSUOpType.ld 294 s0_mask := genWmask(0.U, LSUOpType.ld) 295 // we dont care s0_isFirstIssue and s0_rsIdx and s0_sqIdx in S0 when trying pointchasing 296 // because these signals will be updated in S1 297 s0_isFirstIssue := true.B 298 s0_sqIdx := DontCare 299 } 300 } 301 302 // address align check 303 val addrAligned = LookupTree(s0_uop.fuOpType(1, 0), List( 304 "b00".U -> true.B, //b 305 "b01".U -> (s0_vaddr(0) === 0.U), //h 306 "b10".U -> (s0_vaddr(1, 0) === 0.U), //w 307 "b11".U -> (s0_vaddr(2, 0) === 0.U) //d 308 )) 309 310 311 // accept load flow if dcache ready (dtlb is always ready) 312 // TODO: prefetch need writeback to loadQueueFlag 313 io.out.valid := s0_valid && io.dcacheReq.ready && !io.s0_kill 314 io.out.bits := DontCare 315 io.out.bits.vaddr := s0_vaddr 316 io.out.bits.mask := s0_mask 317 io.out.bits.uop := s0_uop 318 io.out.bits.uop.exceptionVec(loadAddrMisaligned) := !addrAligned 319 io.out.bits.isFirstIssue := s0_isFirstIssue 320 io.out.bits.isPrefetch := isPrefetch 321 io.out.bits.isHWPrefetch := isHWPrefetch 322 io.out.bits.isLoadReplay := lfsrc_loadReplay_select 323 io.out.bits.mshrid := io.replay.bits.mshrid 324 io.out.bits.forward_tlDchannel := io.replay.valid && io.replay.bits.forward_tlDchannel 325 when(io.dtlbReq.valid && s0_isFirstIssue) { 326 io.out.bits.uop.debugInfo.tlbFirstReqTime := GTimer() 327 }.otherwise{ 328 io.out.bits.uop.debugInfo.tlbFirstReqTime := s0_uop.debugInfo.tlbFirstReqTime 329 } 330 io.out.bits.sleepIndex := io.replay.bits.sleepIndex 331 332 // load flow source ready 333 // always accept load flow from load replay queue 334 // io.replay has highest priority 335 io.replay.ready := (io.out.ready && io.dcacheReq.ready && lfsrc_loadReplay_select && !s0_replayShouldWait) 336 337 // accept load flow from rs when: 338 // 1) there is no lsq-replayed load 339 // 2) there is no high confidence prefetch request 340 io.in.ready := (io.out.ready && io.dcacheReq.ready && lfsrc_intloadFirstIssue_select) 341 342 // for hw prefetch load flow feedback, to be added later 343 // io.prefetch_in.ready := lfsrc_hwprefetch_select 344 345 XSDebug(io.dcacheReq.fire, 346 p"[DCACHE LOAD REQ] pc ${Hexadecimal(s0_uop.pc)}, vaddr ${Hexadecimal(s0_vaddr)}\n" 347 ) 348 XSPerfAccumulate("in_valid", io.in.valid) 349 XSPerfAccumulate("in_fire", io.in.fire) 350 XSPerfAccumulate("in_fire_first_issue", s0_valid && s0_isFirstIssue) 351 XSPerfAccumulate("lsq_fire_first_issue", io.replay.fire) 352 XSPerfAccumulate("ldu_fire_first_issue", io.in.fire && io.in.bits.isFirstIssue) 353 XSPerfAccumulate("stall_out", io.out.valid && !io.out.ready && io.dcacheReq.ready) 354 XSPerfAccumulate("stall_dcache", io.out.valid && io.out.ready && !io.dcacheReq.ready) 355 XSPerfAccumulate("addr_spec_success", io.out.fire && s0_vaddr(VAddrBits-1, 12) === io.in.bits.src(0)(VAddrBits-1, 12)) 356 XSPerfAccumulate("addr_spec_failed", io.out.fire && s0_vaddr(VAddrBits-1, 12) =/= io.in.bits.src(0)(VAddrBits-1, 12)) 357 XSPerfAccumulate("addr_spec_success_once", io.out.fire && s0_vaddr(VAddrBits-1, 12) === io.in.bits.src(0)(VAddrBits-1, 12) && io.in.bits.isFirstIssue) 358 XSPerfAccumulate("addr_spec_failed_once", io.out.fire && s0_vaddr(VAddrBits-1, 12) =/= io.in.bits.src(0)(VAddrBits-1, 12) && io.in.bits.isFirstIssue) 359 XSPerfAccumulate("forward_tlDchannel", io.out.bits.forward_tlDchannel) 360 XSPerfAccumulate("hardware_prefetch_fire", io.out.fire && lfsrc_hwprefetch_select) 361 XSPerfAccumulate("software_prefetch_fire", io.out.fire && isPrefetch && lfsrc_intloadFirstIssue_select) 362 XSPerfAccumulate("hardware_prefetch_blocked", io.prefetch_in.valid && !lfsrc_hwprefetch_select) 363 XSPerfAccumulate("hardware_prefetch_total", io.prefetch_in.valid) 364} 365 366// Load Pipeline Stage 1 367// TLB resp (send paddr to dcache) 368class LoadUnit_S1(implicit p: Parameters) extends XSModule with HasCircularQueuePtrHelper { 369 val io = IO(new Bundle() { 370 val in = Flipped(Decoupled(new LqWriteBundle)) 371 val s1_kill = Input(Bool()) 372 val out = Decoupled(new LqWriteBundle) 373 val dtlbResp = Flipped(DecoupledIO(new TlbResp(2))) 374 val lsuPAddr = Output(UInt(PAddrBits.W)) 375 val dcachePAddr = Output(UInt(PAddrBits.W)) 376 val dcacheKill = Output(Bool()) 377 val dcacheBankConflict = Input(Bool()) 378 val fullForwardFast = Output(Bool()) 379 val sbuffer = new LoadForwardQueryIO 380 val lsq = new PipeLoadForwardQueryIO 381 val reExecuteQuery = Flipped(Vec(StorePipelineWidth, Valid(new LoadReExecuteQueryIO))) 382 val csrCtrl = Flipped(new CustomCSRCtrlIO) 383 }) 384 385 val s1_uop = io.in.bits.uop 386 val s1_paddr_dup_lsu = io.dtlbResp.bits.paddr(0) 387 val s1_paddr_dup_dcache = io.dtlbResp.bits.paddr(1) 388 // af & pf exception were modified below. 389 val s1_exception = ExceptionNO.selectByFu(io.out.bits.uop.exceptionVec, LduCfg).asUInt.orR 390 val s1_tlb_miss = io.dtlbResp.bits.miss 391 val s1_mask = io.in.bits.mask 392 val s1_is_prefetch = io.in.bits.isPrefetch 393 val s1_is_hw_prefetch = io.in.bits.isHWPrefetch 394 val s1_is_sw_prefetch = s1_is_prefetch && !s1_is_hw_prefetch 395 val s1_bank_conflict = io.dcacheBankConflict 396 397 io.out.bits := io.in.bits // forwardXX field will be updated in s1 398 399 val s1_tlb_memidx = io.dtlbResp.bits.memidx 400 when(s1_tlb_memidx.is_ld && io.dtlbResp.valid && !s1_tlb_miss && s1_tlb_memidx.idx === io.out.bits.uop.lqIdx.value) { 401 // printf("load idx = %d\n", s1_tlb_memidx.idx) 402 io.out.bits.uop.debugInfo.tlbRespTime := GTimer() 403 } 404 405 io.dtlbResp.ready := true.B 406 407 io.lsuPAddr := s1_paddr_dup_lsu 408 io.dcachePAddr := s1_paddr_dup_dcache 409 //io.dcacheKill := s1_tlb_miss || s1_exception || s1_mmio 410 io.dcacheKill := s1_tlb_miss || s1_exception || io.s1_kill 411 // load forward query datapath 412 io.sbuffer.valid := io.in.valid && !(s1_exception || s1_tlb_miss || io.s1_kill || s1_is_prefetch) 413 io.sbuffer.vaddr := io.in.bits.vaddr 414 io.sbuffer.paddr := s1_paddr_dup_lsu 415 io.sbuffer.uop := s1_uop 416 io.sbuffer.sqIdx := s1_uop.sqIdx 417 io.sbuffer.mask := s1_mask 418 io.sbuffer.pc := s1_uop.pc // FIXME: remove it 419 420 io.lsq.valid := io.in.valid && !(s1_exception || s1_tlb_miss || io.s1_kill || s1_is_prefetch) 421 io.lsq.vaddr := io.in.bits.vaddr 422 io.lsq.paddr := s1_paddr_dup_lsu 423 io.lsq.uop := s1_uop 424 io.lsq.sqIdx := s1_uop.sqIdx 425 io.lsq.sqIdxMask := DontCare // will be overwritten by sqIdxMask pre-generated in s0 426 io.lsq.mask := s1_mask 427 io.lsq.pc := s1_uop.pc // FIXME: remove it 428 429 // st-ld violation query 430 val s1_schedError = VecInit((0 until StorePipelineWidth).map(w => io.reExecuteQuery(w).valid && 431 isAfter(io.in.bits.uop.robIdx, io.reExecuteQuery(w).bits.robIdx) && 432 (s1_paddr_dup_lsu(PAddrBits-1, 3) === io.reExecuteQuery(w).bits.paddr(PAddrBits-1, 3)) && 433 (s1_mask & io.reExecuteQuery(w).bits.mask).orR)).asUInt.orR && !s1_tlb_miss 434 435 // Generate forwardMaskFast to wake up insts earlier 436 val forwardMaskFast = io.lsq.forwardMaskFast.asUInt | io.sbuffer.forwardMaskFast.asUInt 437 io.fullForwardFast := ((~forwardMaskFast).asUInt & s1_mask) === 0.U 438 439 io.out.valid := io.in.valid && !io.s1_kill 440 io.out.bits.paddr := s1_paddr_dup_lsu 441 io.out.bits.tlbMiss := s1_tlb_miss 442 443 // Generate replay signal caused by: 444 // * st-ld violation check 445 // * dcache bank conflict 446 io.out.bits.replayInfo.cause(LoadReplayCauses.schedError) := s1_schedError && !s1_is_sw_prefetch 447 io.out.bits.replayInfo.cause(LoadReplayCauses.bankConflict) := s1_bank_conflict && !s1_is_sw_prefetch 448 io.out.bits.replayInfo.debug := io.in.bits.uop.debugInfo 449 450 // current ori test will cause the case of ldest == 0, below will be modifeid in the future. 451 // af & pf exception were modified 452 io.out.bits.uop.exceptionVec(loadPageFault) := io.dtlbResp.bits.excp(0).pf.ld 453 io.out.bits.uop.exceptionVec(loadAccessFault) := io.dtlbResp.bits.excp(0).af.ld 454 io.out.bits.ptwBack := io.dtlbResp.bits.ptwBack 455 io.out.bits.rsIdx := io.in.bits.rsIdx 456 457 io.in.ready := !io.in.valid || io.out.ready 458 459 XSPerfAccumulate("in_valid", io.in.valid) 460 XSPerfAccumulate("in_fire", io.in.fire) 461 XSPerfAccumulate("in_fire_first_issue", io.in.fire && io.in.bits.isFirstIssue) 462 XSPerfAccumulate("tlb_miss", io.in.fire && s1_tlb_miss) 463 XSPerfAccumulate("tlb_miss_first_issue", io.in.fire && s1_tlb_miss && io.in.bits.isFirstIssue) 464 XSPerfAccumulate("stall_out", io.out.valid && !io.out.ready) 465} 466 467// Load Pipeline Stage 2 468// DCache resp 469class LoadUnit_S2(implicit p: Parameters) extends XSModule 470 with HasLoadHelper 471 with HasCircularQueuePtrHelper 472 with HasDCacheParameters 473{ 474 val io = IO(new Bundle() { 475 val redirect = Flipped(Valid(new Redirect)) 476 val in = Flipped(Decoupled(new LqWriteBundle)) 477 val out = Decoupled(new LqWriteBundle) 478 val dcacheResp = Flipped(DecoupledIO(new DCacheWordResp)) 479 val pmpResp = Flipped(new PMPRespBundle()) 480 val lsq = new LoadForwardQueryIO 481 val dataInvalidSqIdx = Input(new SqPtr) 482 val addrInvalidSqIdx = Input(new SqPtr) 483 val sbuffer = new LoadForwardQueryIO 484 val dataForwarded = Output(Bool()) 485 val fullForward = Output(Bool()) 486 val dcache_kill = Output(Bool()) 487 val loadLoadViolationQueryReq = DecoupledIO(new LoadViolationQueryReq) 488 val storeLoadViolationQueryReq = DecoupledIO(new LoadViolationQueryReq) 489 val csrCtrl = Flipped(new CustomCSRCtrlIO) 490 val static_pm = Input(Valid(Bool())) // valid for static, bits for mmio 491 val loadDataFromDcache = Output(new LoadDataFromDcacheBundle) 492 val reExecuteQuery = Flipped(Vec(StorePipelineWidth, Valid(new LoadReExecuteQueryIO))) 493 // forward tilelink D channel 494 val forward_D = Input(Bool()) 495 val forwardData_D = Input(Vec(8, UInt(8.W))) 496 val sentFastUop = Input(Bool()) 497 // forward mshr data 498 val forward_mshr = Input(Bool()) 499 val forwardData_mshr = Input(Vec(8, UInt(8.W))) 500 501 // indicate whether forward tilelink D channel or mshr data is valid 502 val forward_result_valid = Input(Bool()) 503 504 val feedbackFast = ValidIO(new RSFeedback) 505 val lqReplayFull = Input(Bool()) 506 507 val s2_forward_fail = Output(Bool()) 508 val s2_can_replay_from_fetch = Output(Bool()) // dirty code 509 val s2_dcache_require_replay = Output(Bool()) // dirty code 510 }) 511 512 val pmp = WireInit(io.pmpResp) 513 when (io.static_pm.valid) { 514 pmp.ld := false.B 515 pmp.st := false.B 516 pmp.instr := false.B 517 pmp.mmio := io.static_pm.bits 518 } 519 520 val s2_is_prefetch = io.in.bits.isPrefetch 521 val s2_is_hw_prefetch = io.in.bits.isHWPrefetch 522 523 val forward_D_or_mshr_valid = io.forward_result_valid && (io.forward_D || io.forward_mshr) 524 525 // assert(!reset && io.forward_D && io.forward_mshr && io.in.valid && io.in.bits.forward_tlDchannel, "forward D and mshr at the same time") 526 527 // exception that may cause load addr to be invalid / illegal 528 // 529 // if such exception happen, that inst and its exception info 530 // will be force writebacked to rob 531 val s2_exception_vec = WireInit(io.in.bits.uop.exceptionVec) 532 s2_exception_vec(loadAccessFault) := io.in.bits.uop.exceptionVec(loadAccessFault) || pmp.ld 533 // soft prefetch will not trigger any exception (but ecc error interrupt may be triggered) 534 when (s2_is_prefetch || io.in.bits.tlbMiss) { 535 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 536 } 537 val s2_exception = ExceptionNO.selectByFu(s2_exception_vec, LduCfg).asUInt.orR 538 539 // writeback access fault caused by ecc error / bus error 540 // 541 // * ecc data error is slow to generate, so we will not use it until load stage 3 542 // * in load stage 3, an extra signal io.load_error will be used to 543 544 // now cache ecc error will raise an access fault 545 // at the same time, error info (including error paddr) will be write to 546 // an customized CSR "CACHE_ERROR" 547 // if (EnableAccurateLoadError) { 548 // io.s3_delayed_load_error := io.dcacheResp.bits.error_delayed && 549 // io.csrCtrl.cache_error_enable && 550 // RegNext(io.out.valid) 551 // } else { 552 // io.s3_delayed_load_error := false.B 553 // } 554 555 val actually_mmio = pmp.mmio 556 val s2_uop = io.in.bits.uop 557 val s2_mask = io.in.bits.mask 558 val s2_paddr = io.in.bits.paddr 559 val s2_tlb_miss = io.in.bits.tlbMiss 560 val s2_mmio = !s2_is_prefetch && actually_mmio && !s2_exception && !s2_tlb_miss 561 val s2_cache_miss = io.dcacheResp.bits.miss && !forward_D_or_mshr_valid 562 val s2_cache_replay = io.dcacheResp.bits.replay && !forward_D_or_mshr_valid 563 val s2_cache_tag_error = RegNext(io.csrCtrl.cache_error_enable) && io.dcacheResp.bits.tag_error 564 val s2_forward_fail = io.lsq.matchInvalid || io.sbuffer.matchInvalid 565 val s2_wait_store = WireInit(false.B) 566 val s2_data_invalid = io.lsq.dataInvalid && !s2_exception 567 val s2_fullForward = WireInit(false.B) 568 569 570 io.s2_forward_fail := s2_forward_fail 571 io.dcache_kill := pmp.ld || pmp.mmio // move pmp resp kill to outside 572 io.dcacheResp.ready := true.B 573 val dcacheShouldResp = !(s2_tlb_miss || s2_exception || s2_mmio || s2_is_prefetch) 574 assert(!(io.in.valid && (dcacheShouldResp && !io.dcacheResp.valid)), "DCache response got lost") 575 576 // st-ld violation query 577 // NeedFastRecovery Valid when 578 // 1. Fast recovery query request Valid. 579 // 2. Load instruction is younger than requestors(store instructions). 580 // 3. Physical address match. 581 // 4. Data contains. 582 val s2_schedError = VecInit((0 until StorePipelineWidth).map(w => io.reExecuteQuery(w).valid && 583 isAfter(io.in.bits.uop.robIdx, io.reExecuteQuery(w).bits.robIdx) && 584 (s2_paddr(PAddrBits-1,3) === io.reExecuteQuery(w).bits.paddr(PAddrBits-1, 3)) && 585 (s2_mask & io.reExecuteQuery(w).bits.mask).orR)).asUInt.orR && 586 !s2_tlb_miss 587 588 // need allocate new entry 589 val s2_allocValid = !s2_tlb_miss && 590 !s2_is_prefetch && 591 !s2_exception && 592 !s2_mmio && 593 !s2_wait_store && 594 !io.in.bits.replayInfo.cause(LoadReplayCauses.schedError) 595 596 // ld-ld violation require 597 io.loadLoadViolationQueryReq.valid := io.in.valid && s2_allocValid 598 io.loadLoadViolationQueryReq.bits.uop := io.in.bits.uop 599 io.loadLoadViolationQueryReq.bits.mask := s2_mask 600 io.loadLoadViolationQueryReq.bits.paddr := s2_paddr 601 if (EnableFastForward) { 602 io.loadLoadViolationQueryReq.bits.datavalid := Mux(s2_fullForward, true.B, !s2_cache_miss) && !io.s2_dcache_require_replay 603 } else { 604 io.loadLoadViolationQueryReq.bits.datavalid := Mux(s2_fullForward, true.B, !s2_cache_miss) 605 } 606 607 // st-ld violation require 608 io.storeLoadViolationQueryReq.valid := io.in.valid && s2_allocValid 609 io.storeLoadViolationQueryReq.bits.uop := io.in.bits.uop 610 io.storeLoadViolationQueryReq.bits.mask := s2_mask 611 io.storeLoadViolationQueryReq.bits.paddr := s2_paddr 612 io.storeLoadViolationQueryReq.bits.datavalid := io.loadLoadViolationQueryReq.bits.datavalid 613 614 val s2_rarCanAccept = !io.loadLoadViolationQueryReq.valid || io.loadLoadViolationQueryReq.ready 615 val s2_rawCanAccept = !io.storeLoadViolationQueryReq.valid || io.storeLoadViolationQueryReq.ready 616 val s2_rejectEnq = !s2_rarCanAccept || !s2_rawCanAccept 617 618 // merge forward result 619 // lsq has higher priority than sbuffer 620 val forwardMask = Wire(Vec(8, Bool())) 621 val forwardData = Wire(Vec(8, UInt(8.W))) 622 623 val fullForward = ((~forwardMask.asUInt).asUInt & s2_mask) === 0.U && !io.lsq.dataInvalid 624 io.lsq := DontCare 625 io.sbuffer := DontCare 626 io.fullForward := fullForward 627 s2_fullForward := fullForward 628 629 // generate XLEN/8 Muxs 630 for (i <- 0 until XLEN / 8) { 631 forwardMask(i) := io.lsq.forwardMask(i) || io.sbuffer.forwardMask(i) 632 forwardData(i) := Mux(io.lsq.forwardMask(i), io.lsq.forwardData(i), io.sbuffer.forwardData(i)) 633 } 634 635 XSDebug(io.out.fire, "[FWD LOAD RESP] pc %x fwd %x(%b) + %x(%b)\n", 636 s2_uop.pc, 637 io.lsq.forwardData.asUInt, io.lsq.forwardMask.asUInt, 638 io.in.bits.forwardData.asUInt, io.in.bits.forwardMask.asUInt 639 ) 640 641 // data merge 642 // val rdataVec = VecInit((0 until XLEN / 8).map(j => 643 // Mux(forwardMask(j), forwardData(j), io.dcacheResp.bits.data(8*(j+1)-1, 8*j)) 644 // )) // s2_rdataVec will be write to load queue 645 // val rdata = rdataVec.asUInt 646 // val rdataSel = LookupTree(s2_paddr(2, 0), List( 647 // "b000".U -> rdata(63, 0), 648 // "b001".U -> rdata(63, 8), 649 // "b010".U -> rdata(63, 16), 650 // "b011".U -> rdata(63, 24), 651 // "b100".U -> rdata(63, 32), 652 // "b101".U -> rdata(63, 40), 653 // "b110".U -> rdata(63, 48), 654 // "b111".U -> rdata(63, 56) 655 // )) 656 // val rdataPartialLoad = rdataHelper(s2_uop, rdataSel) // s2_rdataPartialLoad is not used 657 io.feedbackFast.valid := io.in.valid && !io.in.bits.isLoadReplay && !s2_exception && io.lqReplayFull && io.out.bits.replayInfo.needReplay() && !io.out.bits.uop.robIdx.needFlush(io.redirect) 658 io.feedbackFast.bits.hit := false.B 659 io.feedbackFast.bits.flushState := io.in.bits.ptwBack 660 io.feedbackFast.bits.rsIdx := io.in.bits.rsIdx 661 io.feedbackFast.bits.sourceType := RSFeedbackType.lrqFull 662 io.feedbackFast.bits.dataInvalidSqIdx := DontCare 663 664 io.out.valid := io.in.valid && !io.feedbackFast.valid && !s2_is_hw_prefetch // hardware prefetch flow should not be writebacked 665 // write_lq_safe is needed by dup logic 666 // io.write_lq_safe := !s2_tlb_miss && !s2_data_invalid 667 // Inst will be canceled in store queue / lsq, 668 // so we do not need to care about flush in load / store unit's out.valid 669 io.out.bits := io.in.bits 670 // io.out.bits.data := rdataPartialLoad 671 io.out.bits.data := 0.U // data will be generated in load_s3 672 // when exception occurs, set it to not miss and let it write back to rob (via int port) 673 if (EnableFastForward) { 674 io.out.bits.miss := s2_cache_miss && 675 !fullForward && 676 !s2_exception && 677 !s2_is_prefetch && 678 !s2_mmio 679 } else { 680 io.out.bits.miss := s2_cache_miss && 681 !s2_exception && 682 !s2_is_prefetch && 683 !s2_mmio 684 } 685 io.out.bits.uop.fpWen := io.in.bits.uop.fpWen && !s2_exception 686 687 // val s2_loadDataFromDcache = new LoadDataFromDcacheBundle 688 // s2_loadDataFromDcache.forwardMask := forwardMask 689 // s2_loadDataFromDcache.forwardData := forwardData 690 // s2_loadDataFromDcache.uop := io.out.bits.uop 691 // s2_loadDataFromDcache.addrOffset := s2_paddr(2, 0) 692 // // forward D or mshr 693 // s2_loadDataFromDcache.forward_D := io.forward_D 694 // s2_loadDataFromDcache.forwardData_D := io.forwardData_D 695 // s2_loadDataFromDcache.forward_mshr := io.forward_mshr 696 // s2_loadDataFromDcache.forwardData_mshr := io.forwardData_mshr 697 // s2_loadDataFromDcache.forward_result_valid := io.forward_result_valid 698 // io.loadDataFromDcache := RegEnable(s2_loadDataFromDcache, io.in.valid) 699 io.loadDataFromDcache.respDcacheData := io.dcacheResp.bits.data_delayed 700 io.loadDataFromDcache.forwardMask := RegEnable(forwardMask, io.in.valid) 701 io.loadDataFromDcache.forwardData := RegEnable(forwardData, io.in.valid) 702 io.loadDataFromDcache.uop := RegEnable(io.out.bits.uop, io.in.valid) 703 io.loadDataFromDcache.addrOffset := RegEnable(s2_paddr(2, 0), io.in.valid) 704 // forward D or mshr 705 io.loadDataFromDcache.forward_D := RegEnable(io.forward_D, io.in.valid) 706 io.loadDataFromDcache.forwardData_D := RegEnable(io.forwardData_D, io.in.valid) 707 io.loadDataFromDcache.forward_mshr := RegEnable(io.forward_mshr, io.in.valid) 708 io.loadDataFromDcache.forwardData_mshr := RegEnable(io.forwardData_mshr, io.in.valid) 709 io.loadDataFromDcache.forward_result_valid := RegEnable(io.forward_result_valid, io.in.valid) 710 711 io.s2_can_replay_from_fetch := !s2_mmio && !s2_is_prefetch && !s2_tlb_miss 712 // if forward fail, replay this inst from fetch 713 val debug_forwardFailReplay = s2_forward_fail && !s2_mmio && !s2_is_prefetch && !s2_tlb_miss 714 // if ld-ld violation is detected, replay from this inst from fetch 715 val debug_ldldVioReplay = false.B // s2_ldld_violation && !s2_mmio && !s2_is_prefetch && !s2_tlb_miss 716 // io.out.bits.uop.ctrl.replayInst := false.B 717 718 io.out.bits.mmio := s2_mmio 719 io.out.bits.uop.flushPipe := io.sentFastUop && s2_mmio // remove io.sentFastUop 720 io.out.bits.uop.exceptionVec := s2_exception_vec // cache error not included 721 722 // For timing reasons, sometimes we can not let 723 // io.out.bits.miss := s2_cache_miss && !s2_exception && !fullForward 724 // We use io.dataForwarded instead. It means: 725 // 1. Forward logic have prepared all data needed, 726 // and dcache query is no longer needed. 727 // 2. ... or data cache tag error is detected, this kind of inst 728 // will not update miss queue. That is to say, if miss, that inst 729 // may not be refilled 730 // Such inst will be writebacked from load queue. 731 io.dataForwarded := s2_cache_miss && !s2_exception && 732 (fullForward || s2_cache_tag_error) 733 // io.out.bits.forwardX will be send to lq 734 io.out.bits.forwardMask := forwardMask 735 // data from dcache is not included in io.out.bits.forwardData 736 io.out.bits.forwardData := forwardData 737 738 io.in.ready := io.out.ready || !io.in.valid 739 740 // Generate replay signal caused by: 741 // * st-ld violation check 742 // * tlb miss 743 // * dcache replay 744 // * forward data invalid 745 // * dcache miss 746 io.out.bits.replayInfo.cause(LoadReplayCauses.waitStore) := s2_wait_store && !s2_mmio && !s2_is_prefetch 747 io.out.bits.replayInfo.cause(LoadReplayCauses.tlbMiss) := s2_tlb_miss 748 io.out.bits.replayInfo.cause(LoadReplayCauses.schedError) := (io.in.bits.replayInfo.cause(LoadReplayCauses.schedError) || s2_schedError) && !s2_mmio && !s2_is_prefetch 749 io.out.bits.replayInfo.cause(LoadReplayCauses.dcacheMiss) := io.out.bits.miss 750 if (EnableFastForward) { 751 io.out.bits.replayInfo.cause(LoadReplayCauses.dcacheReplay) := !(!s2_cache_replay || s2_is_prefetch || s2_mmio || s2_exception || fullForward) 752 }else { 753 io.out.bits.replayInfo.cause(LoadReplayCauses.dcacheReplay) := !(!s2_cache_replay || s2_is_prefetch || s2_mmio || s2_exception || io.dataForwarded) 754 } 755 io.out.bits.replayInfo.cause(LoadReplayCauses.forwardFail) := s2_data_invalid && !s2_mmio && !s2_is_prefetch 756 io.out.bits.replayInfo.cause(LoadReplayCauses.rejectEnq) := s2_rejectEnq && !s2_mmio && !s2_is_prefetch && !s2_exception 757 io.out.bits.replayInfo.canForwardFullData := io.dataForwarded 758 io.out.bits.replayInfo.dataInvalidSqIdx := io.dataInvalidSqIdx 759 io.out.bits.replayInfo.addrInvalidSqIdx := io.addrInvalidSqIdx // io.in.bits.uop.sqIdx - io.oracleMDPQuery.resp.distance // io.addrInvalidSqIdx 760 io.out.bits.replayInfo.replayCarry := io.dcacheResp.bits.replayCarry 761 io.out.bits.replayInfo.missMSHRId := io.dcacheResp.bits.mshr_id 762 io.out.bits.replayInfo.dataInLastBeat := io.in.bits.paddr(log2Up(refillBytes)) 763 io.out.bits.replayInfo.debug := io.in.bits.uop.debugInfo 764 765 // To be removed 766 val s2_need_replay_from_rs = WireInit(false.B) 767 // s2_cache_replay is quite slow to generate, send it separately to LQ 768 if (EnableFastForward) { 769 io.s2_dcache_require_replay := s2_cache_replay && !fullForward 770 } else { 771 io.s2_dcache_require_replay := s2_cache_replay && 772 s2_need_replay_from_rs && 773 !io.dataForwarded && 774 !s2_is_prefetch && 775 io.out.bits.miss 776 } 777 778 XSPerfAccumulate("in_valid", io.in.valid) 779 XSPerfAccumulate("in_fire", io.in.fire) 780 XSPerfAccumulate("in_fire_first_issue", io.in.fire && io.in.bits.isFirstIssue) 781 XSPerfAccumulate("dcache_miss", io.in.fire && s2_cache_miss) 782 XSPerfAccumulate("dcache_miss_first_issue", io.in.fire && s2_cache_miss && io.in.bits.isFirstIssue) 783 XSPerfAccumulate("full_forward", io.in.valid && fullForward) 784 XSPerfAccumulate("dcache_miss_full_forward", io.in.valid && s2_cache_miss && fullForward) 785 XSPerfAccumulate("stall_out", io.out.valid && !io.out.ready) 786 XSPerfAccumulate("prefetch", io.in.fire && s2_is_prefetch) 787 XSPerfAccumulate("prefetch_ignored", io.in.fire && s2_is_prefetch && s2_cache_replay) // ignore prefetch for mshr full / miss req port conflict 788 XSPerfAccumulate("prefetch_miss", io.in.fire && s2_is_prefetch && s2_cache_miss) // prefetch req miss in l1 789 XSPerfAccumulate("prefetch_hit", io.in.fire && s2_is_prefetch && !s2_cache_miss) // prefetch req hit in l1 790 // prefetch a missed line in l1, and l1 accepted it 791 XSPerfAccumulate("prefetch_accept", io.in.fire && s2_is_prefetch && s2_cache_miss && !s2_cache_replay) 792} 793 794class LoadUnit(implicit p: Parameters) extends XSModule 795 with HasLoadHelper 796 with HasPerfEvents 797 with HasDCacheParameters 798 with HasCircularQueuePtrHelper 799{ 800 val io = IO(new Bundle() { 801 val loadIn = Flipped(Decoupled(new MemExuInput)) 802 val loadOut = Decoupled(new MemExuOutput) 803 val redirect = Flipped(ValidIO(new Redirect)) 804 val dcache = new DCacheLoadIO 805 val sbuffer = new LoadForwardQueryIO 806 val lsq = new LoadToLsqIO 807 val tlDchannel = Input(new DcacheToLduForwardIO) 808 val forward_mshr = Flipped(new LduToMissqueueForwardIO) 809 val refill = Flipped(ValidIO(new Refill)) 810 val fastUop = ValidIO(new DynInst) // early wakeup signal generated in load_s1, send to RS in load_s2 811 val trigger = Vec(3, new LoadUnitTriggerIO) 812 813 val tlb = new TlbRequestIO(2) 814 val pmp = Flipped(new PMPRespBundle()) // arrive same to tlb now 815 816 // provide prefetch info 817 val prefetch_train = ValidIO(new LdPrefetchTrainBundle()) 818 819 // hardware prefetch to l1 cache req 820 val prefetch_req = Flipped(ValidIO(new L1PrefetchReq)) 821 822 // load to load fast path 823 val fastpathOut = Output(new LoadToLoadIO) 824 val fastpathIn = Input(new LoadToLoadIO) 825 val loadFastMatch = Input(Bool()) 826 val loadFastImm = Input(UInt(12.W)) 827 828 // rs feedback 829 val feedbackFast = ValidIO(new RSFeedback) // stage 2 830 val feedbackSlow = ValidIO(new RSFeedback) // stage 3 831 832 // load ecc 833 val s3_delayedLoadError = Output(Bool()) // load ecc error 834 // Note that io.s3_delayed_load_error and io.lsq.s3_delayed_load_error is different 835 836 // load unit ctrl 837 val csrCtrl = Flipped(new CustomCSRCtrlIO) 838 839 val reExecuteQuery = Flipped(Vec(StorePipelineWidth, Valid(new LoadReExecuteQueryIO))) // load replay 840 val replay = Flipped(Decoupled(new LsPipelineBundle)) 841 val debug_ls = Output(new DebugLsInfoBundle) 842 val s2IsPointerChasing = Output(Bool()) // provide right pc for hw prefetch 843 val lqReplayFull = Input(Bool()) 844 }) 845 846 val load_s0 = Module(new LoadUnit_S0) 847 val load_s1 = Module(new LoadUnit_S1) 848 val load_s2 = Module(new LoadUnit_S2) 849 850 // load s0 851 load_s0.io.in <> io.loadIn 852 load_s0.io.dtlbReq <> io.tlb.req 853 load_s0.io.dcacheReq <> io.dcache.req 854 load_s0.io.s0_kill := false.B 855 load_s0.io.replay <> io.replay 856 // hareware prefetch to l1 857 load_s0.io.prefetch_in <> io.prefetch_req 858 859 // we try pointerchasing if lfsrc_l2lForward_select condition is satisfied 860 val s0_tryPointerChasing = load_s0.io.l2lForward_select 861 val s0_pointerChasingVAddr = io.fastpathIn.data(5, 0) +& io.loadFastImm(5, 0) 862 load_s0.io.fastpath.valid := io.fastpathIn.valid 863 load_s0.io.fastpath.data := Cat(io.fastpathIn.data(XLEN-1, 6), s0_pointerChasingVAddr(5,0)) 864 865 val s1_data = PipelineConnect(load_s0.io.out, load_s1.io.in, true.B, 866 load_s0.io.out.bits.uop.robIdx.needFlush(io.redirect) && !s0_tryPointerChasing).get 867 868 // load s1 869 // update s1_kill when any source has valid request 870 load_s1.io.s1_kill := RegEnable(load_s0.io.s0_kill, false.B, io.loadIn.valid || io.replay.valid || io.fastpathIn.valid) 871 io.tlb.req_kill := load_s1.io.s1_kill 872 load_s1.io.dtlbResp <> io.tlb.resp 873 load_s1.io.lsuPAddr <> io.dcache.s1_paddr_dup_lsu 874 load_s1.io.dcachePAddr <> io.dcache.s1_paddr_dup_dcache 875 load_s1.io.dcacheKill <> io.dcache.s1_kill 876 load_s1.io.sbuffer <> io.sbuffer 877 load_s1.io.lsq <> io.lsq.forward 878 load_s1.io.dcacheBankConflict <> io.dcache.s1_bank_conflict 879 load_s1.io.csrCtrl <> io.csrCtrl 880 load_s1.io.reExecuteQuery := io.reExecuteQuery 881 882 // when S0 has opportunity to try pointerchasing, make sure it truely goes to S1 883 // which is S0's out is ready and dcache is ready 884 val s0_doTryPointerChasing = s0_tryPointerChasing && load_s0.io.out.ready && load_s0.io.dcacheReq.ready 885 val s1_tryPointerChasing = RegNext(s0_doTryPointerChasing, false.B) 886 val s1_pointerChasingVAddr = RegEnable(s0_pointerChasingVAddr, s0_doTryPointerChasing) 887 val cancelPointerChasing = WireInit(false.B) 888 if (EnableLoadToLoadForward) { 889 // Sometimes, we need to cancel the load-load forwarding. 890 // These can be put at S0 if timing is bad at S1. 891 // Case 0: CACHE_SET(base + offset) != CACHE_SET(base) (lowest 6-bit addition has an overflow) 892 val addressMisMatch = s1_pointerChasingVAddr(6) || RegEnable(io.loadFastImm(11, 6).orR, s0_doTryPointerChasing) 893 // Case 1: the address is not 64-bit aligned or the fuOpType is not LD 894 val addressNotAligned = s1_pointerChasingVAddr(2, 0).orR 895 val fuOpTypeIsNotLd = io.loadIn.bits.uop.fuOpType =/= LSUOpType.ld 896 // Case 2: this is not a valid load-load pair 897 val notFastMatch = RegEnable(!io.loadFastMatch, s0_tryPointerChasing) 898 // Case 3: this load-load uop is cancelled 899 val isCancelled = !io.loadIn.valid 900 when (s1_tryPointerChasing) { 901 cancelPointerChasing := addressMisMatch || addressNotAligned || fuOpTypeIsNotLd || notFastMatch || isCancelled 902 load_s1.io.in.bits.uop := io.loadIn.bits.uop 903 val spec_vaddr = s1_data.vaddr 904 val vaddr = Cat(spec_vaddr(VAddrBits - 1, 6), s1_pointerChasingVAddr(5, 3), 0.U(3.W)) 905 load_s1.io.in.bits.vaddr := vaddr 906 load_s1.io.in.bits.rsIdx := io.loadIn.bits.iqIdx 907 load_s1.io.in.bits.isFirstIssue := io.loadIn.bits.isFirstIssue 908 // We need to replace vaddr(5, 3). 909 val spec_paddr = io.tlb.resp.bits.paddr(0) 910 load_s1.io.dtlbResp.bits.paddr.foreach(_ := Cat(spec_paddr(PAddrBits - 1, 6), s1_pointerChasingVAddr(5, 3), 0.U(3.W))) 911 // recored tlb time when get the data to ensure the correctness of the latency calculation (although it should not record in here, because it does not use tlb) 912 load_s1.io.in.bits.uop.debugInfo.tlbFirstReqTime := GTimer() 913 load_s1.io.in.bits.uop.debugInfo.tlbRespTime := GTimer() 914 } 915 when (cancelPointerChasing) { 916 load_s1.io.s1_kill := true.B 917 }.otherwise { 918 load_s0.io.s0_kill := s1_tryPointerChasing && !io.replay.fire 919 when (s1_tryPointerChasing) { 920 io.loadIn.ready := true.B 921 } 922 } 923 924 XSPerfAccumulate("load_to_load_forward", s1_tryPointerChasing && !cancelPointerChasing) 925 XSPerfAccumulate("load_to_load_forward_try", s1_tryPointerChasing) 926 XSPerfAccumulate("load_to_load_forward_fail", cancelPointerChasing) 927 XSPerfAccumulate("load_to_load_forward_fail_cancelled", cancelPointerChasing && isCancelled) 928 XSPerfAccumulate("load_to_load_forward_fail_wakeup_mismatch", cancelPointerChasing && !isCancelled && notFastMatch) 929 XSPerfAccumulate("load_to_load_forward_fail_op_not_ld", 930 cancelPointerChasing && !isCancelled && !notFastMatch && fuOpTypeIsNotLd) 931 XSPerfAccumulate("load_to_load_forward_fail_addr_align", 932 cancelPointerChasing && !isCancelled && !notFastMatch && !fuOpTypeIsNotLd && addressNotAligned) 933 XSPerfAccumulate("load_to_load_forward_fail_set_mismatch", 934 cancelPointerChasing && !isCancelled && !notFastMatch && !fuOpTypeIsNotLd && !addressNotAligned && addressMisMatch) 935 } 936 PipelineConnect(load_s1.io.out, load_s2.io.in, true.B, 937 load_s1.io.out.bits.uop.robIdx.needFlush(io.redirect) || cancelPointerChasing) 938 939 val (forward_D, forwardData_D) = io.tlDchannel.forward(load_s1.io.out.valid && load_s1.io.out.bits.forward_tlDchannel, load_s1.io.out.bits.mshrid, load_s1.io.out.bits.paddr) 940 941 io.forward_mshr.valid := load_s1.io.out.valid && load_s1.io.out.bits.forward_tlDchannel 942 io.forward_mshr.mshrid := load_s1.io.out.bits.mshrid 943 io.forward_mshr.paddr := load_s1.io.out.bits.paddr 944 val (forward_result_valid, forward_mshr, forwardData_mshr) = io.forward_mshr.forward() 945 946 XSPerfAccumulate("successfully_forward_channel_D", forward_D && forward_result_valid) 947 XSPerfAccumulate("successfully_forward_mshr", forward_mshr && forward_result_valid) 948 949 // load s2 950 load_s2.io.redirect <> io.redirect 951 load_s2.io.forward_D := forward_D 952 load_s2.io.forwardData_D := forwardData_D 953 load_s2.io.forward_result_valid := forward_result_valid 954 load_s2.io.forward_mshr := forward_mshr 955 load_s2.io.forwardData_mshr := forwardData_mshr 956 io.s2IsPointerChasing := RegEnable(s1_tryPointerChasing && !cancelPointerChasing, load_s1.io.out.fire) 957 io.prefetch_train.bits.fromLsPipelineBundle(load_s2.io.in.bits) 958 // override miss bit 959 io.prefetch_train.bits.miss := io.dcache.resp.bits.miss 960 io.prefetch_train.bits.meta_prefetch := io.dcache.resp.bits.meta_prefetch 961 io.prefetch_train.bits.meta_access := io.dcache.resp.bits.meta_access 962 io.prefetch_train.valid := load_s2.io.in.fire && !load_s2.io.out.bits.mmio && !load_s2.io.in.bits.tlbMiss 963 io.dcache.s2_kill := load_s2.io.dcache_kill // to kill mmio resp which are redirected 964 if (env.FPGAPlatform) 965 io.dcache.s2_pc := DontCare 966 else 967 io.dcache.s2_pc := load_s2.io.out.bits.uop.pc 968 load_s2.io.dcacheResp <> io.dcache.resp 969 load_s2.io.pmpResp <> io.pmp 970 load_s2.io.static_pm := RegNext(io.tlb.resp.bits.static_pm) 971 load_s2.io.lsq.forwardData <> io.lsq.forward.forwardData 972 load_s2.io.lsq.forwardMask <> io.lsq.forward.forwardMask 973 load_s2.io.lsq.forwardMaskFast <> io.lsq.forward.forwardMaskFast // should not be used in load_s2 974 load_s2.io.lsq.dataInvalid <> io.lsq.forward.dataInvalid 975 load_s2.io.lsq.matchInvalid <> io.lsq.forward.matchInvalid 976 load_s2.io.lsq.addrInvalid <> io.lsq.forward.addrInvalid 977 load_s2.io.sbuffer.forwardData <> io.sbuffer.forwardData 978 load_s2.io.sbuffer.forwardMask <> io.sbuffer.forwardMask 979 load_s2.io.sbuffer.forwardMaskFast <> io.sbuffer.forwardMaskFast // should not be used in load_s2 980 load_s2.io.sbuffer.dataInvalid <> io.sbuffer.dataInvalid // always false 981 load_s2.io.sbuffer.matchInvalid <> io.sbuffer.matchInvalid 982 load_s2.io.sbuffer.addrInvalid := DontCare // useless 983 load_s2.io.dataInvalidSqIdx <> io.lsq.forward.dataInvalidSqIdx // provide dataInvalidSqIdx to make wakeup faster 984 load_s2.io.addrInvalidSqIdx <> io.lsq.forward.addrInvalidSqIdx // provide addrInvalidSqIdx to make wakeup faster 985 load_s2.io.csrCtrl <> io.csrCtrl 986 load_s2.io.sentFastUop := io.fastUop.valid 987 load_s2.io.reExecuteQuery := io.reExecuteQuery 988 load_s2.io.loadLoadViolationQueryReq <> io.lsq.loadLoadViolationQuery.req 989 load_s2.io.storeLoadViolationQueryReq <> io.lsq.storeLoadViolationQuery.req 990 load_s2.io.feedbackFast <> io.feedbackFast 991 load_s2.io.lqReplayFull <> io.lqReplayFull 992 993 994 995 996 // pre-calcuate sqIdx mask in s0, then send it to lsq in s1 for forwarding 997 val sqIdxMaskReg = RegNext(UIntToMask(load_s0.io.s0_sqIdx.value, StoreQueueSize)) 998 // to enable load-load, sqIdxMask must be calculated based on loadIn.uop 999 // If the timing here is not OK, load-load forwarding has to be disabled. 1000 // Or we calculate sqIdxMask at RS?? 1001 io.lsq.forward.sqIdxMask := sqIdxMaskReg 1002 if (EnableLoadToLoadForward) { 1003 when (s1_tryPointerChasing) { 1004 io.lsq.forward.sqIdxMask := UIntToMask(io.loadIn.bits.uop.sqIdx.value, StoreQueueSize) 1005 } 1006 } 1007 1008 // // use s2_hit_way to select data received in s1 1009 // load_s2.io.dcacheResp.bits.data := Mux1H(RegNext(io.dcache.s1_hit_way), RegNext(io.dcache.s1_data)) 1010 // assert(load_s2.io.dcacheResp.bits.data === io.dcache.resp.bits.data) 1011 1012 // now io.fastUop.valid is sent to RS in load_s2 1013 // val forward_D_or_mshr_valid = forward_result_valid && (forward_D || forward_mshr) 1014 // val s2_dcache_hit = io.dcache.s2_hit || forward_D_or_mshr_valid // dcache hit dup in lsu side 1015 1016 // never fast wakeup 1017 val forward_D_or_mshr_valid = forward_result_valid && (forward_D || forward_mshr) 1018 val s2_dcache_hit = io.dcache.s2_hit || forward_D_or_mshr_valid // dcache hit dup in lsu side 1019 1020 io.fastUop.valid := RegNext( 1021 !io.dcache.s1_disable_fast_wakeup && // load fast wakeup should be disabled when dcache data read is not ready 1022 load_s1.io.in.valid && // valid load request 1023 !load_s1.io.s1_kill && // killed by load-load forwarding 1024 !load_s1.io.dtlbResp.bits.fast_miss && // not mmio or tlb miss, pf / af not included here 1025 !io.lsq.forward.dataInvalidFast // forward failed 1026 ) && 1027 !RegNext(load_s1.io.out.bits.uop.robIdx.needFlush(io.redirect)) && 1028 (load_s2.io.in.valid && s2_dcache_hit && !load_s2.io.out.bits.replayInfo.needReplay()) 1029 io.fastUop.bits := RegNext(load_s1.io.out.bits.uop) 1030 1031 XSDebug(load_s0.io.out.valid, 1032 p"S0: pc ${Hexadecimal(load_s0.io.out.bits.uop.pc)}, lId ${Hexadecimal(load_s0.io.out.bits.uop.lqIdx.asUInt)}, " + 1033 p"vaddr ${Hexadecimal(load_s0.io.out.bits.vaddr)}, mask ${Hexadecimal(load_s0.io.out.bits.mask)}\n") 1034 XSDebug(load_s1.io.out.valid, 1035 p"S1: pc ${Hexadecimal(load_s1.io.out.bits.uop.pc)}, lId ${Hexadecimal(load_s1.io.out.bits.uop.lqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 1036 p"paddr ${Hexadecimal(load_s1.io.out.bits.paddr)}, mmio ${load_s1.io.out.bits.mmio}\n") 1037 1038 // load s2 1039 load_s2.io.out.ready := true.B 1040 val s2_loadOutValid = load_s2.io.out.valid 1041 // generate duplicated load queue data wen 1042 val s2_loadValidVec = RegInit(0.U(6.W)) 1043 val s2_loadLeftFire = load_s1.io.out.valid && load_s2.io.in.ready 1044 // val write_lq_safe = load_s2.io.write_lq_safe 1045 s2_loadValidVec := 0x0.U(6.W) 1046 when (s2_loadLeftFire && !load_s1.io.out.bits.isHWPrefetch) { s2_loadValidVec := 0x3f.U(6.W) } // TODO: refactor me 1047 when (load_s1.io.out.bits.uop.robIdx.needFlush(io.redirect)) { s2_loadValidVec := 0x0.U(6.W) } 1048 assert(RegNext((load_s2.io.in.valid === s2_loadValidVec(0)) || RegNext(load_s1.io.out.bits.isHWPrefetch))) 1049 1050 // load s3 1051 // writeback to LSQ 1052 // Current dcache use MSHR 1053 // Load queue will be updated at s2 for both hit/miss int/fp load 1054 val s3_loadOutBits = RegEnable(load_s2.io.out.bits, s2_loadOutValid) 1055 val s3_loadOutValid = RegNext(s2_loadOutValid) && !RegNext(load_s2.io.out.bits.uop.robIdx.needFlush(io.redirect)) 1056 io.lsq.loadIn.valid := s3_loadOutValid 1057 io.lsq.loadIn.bits := s3_loadOutBits 1058 1059 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1060 1061 // make chisel happy 1062 val s3_loadValidVec = Reg(UInt(6.W)) 1063 s3_loadValidVec := s2_loadValidVec 1064 io.lsq.loadIn.bits.lqDataWenDup := s3_loadValidVec.asBools 1065 1066 // s2_dcache_require_replay signal will be RegNexted, then used in s3 1067 val s3_dcacheRequireReplay = RegNext(load_s2.io.s2_dcache_require_replay) 1068 val s3_delayedLoadError = 1069 if (EnableAccurateLoadError) { 1070 io.dcache.resp.bits.error_delayed && RegNext(io.csrCtrl.cache_error_enable) 1071 } else { 1072 WireInit(false.B) 1073 } 1074 val s3_canReplayFromFetch = RegNext(load_s2.io.s2_can_replay_from_fetch) 1075 io.s3_delayedLoadError := false.B // s3_delayedLoadError 1076 io.lsq.loadIn.bits.dcacheRequireReplay := s3_dcacheRequireReplay 1077 1078 1079 val s3_vpMatchInvalid = RegNext(io.lsq.forward.matchInvalid || io.sbuffer.matchInvalid) 1080 val s3_ldld_replayFromFetch = 1081 io.lsq.loadLoadViolationQuery.resp.valid && 1082 io.lsq.loadLoadViolationQuery.resp.bits.replayFromFetch && 1083 RegNext(io.csrCtrl.ldld_vio_check_enable) 1084 1085 // write to rob and writeback bus 1086 val s3_replayInfo = s3_loadOutBits.replayInfo 1087 val s3_replayInst = s3_vpMatchInvalid || s3_ldld_replayFromFetch 1088 val s3_selReplayCause = PriorityEncoderOH(s3_replayInfo.cause.asUInt) 1089 dontTouch(s3_selReplayCause) // for debug 1090 val s3_forceReplay = s3_selReplayCause(LoadReplayCauses.schedError) || 1091 s3_selReplayCause(LoadReplayCauses.tlbMiss) || 1092 s3_selReplayCause(LoadReplayCauses.waitStore) 1093 1094 val s3_exception = ExceptionNO.selectByFu(s3_loadOutBits.uop.exceptionVec, LduCfg).asUInt.orR 1095 when ((s3_exception || s3_delayedLoadError || s3_replayInst) && !s3_forceReplay) { 1096 io.lsq.loadIn.bits.replayInfo.cause := 0.U.asTypeOf(s3_replayInfo.cause.cloneType) 1097 } .otherwise { 1098 io.lsq.loadIn.bits.replayInfo.cause := VecInit(s3_selReplayCause.asBools) 1099 } 1100 dontTouch(io.lsq.loadIn.bits.replayInfo.cause) 1101 1102 1103 1104 // Int load, if hit, will be writebacked at s2 1105 val hitLoadOut = Wire(Valid(new MemExuOutput)) 1106 hitLoadOut.valid := s3_loadOutValid && !io.lsq.loadIn.bits.replayInfo.needReplay() && !s3_loadOutBits.mmio 1107 hitLoadOut.bits.uop := s3_loadOutBits.uop 1108 hitLoadOut.bits.uop.exceptionVec(loadAccessFault) := s3_delayedLoadError && !s3_loadOutBits.tlbMiss || 1109 s3_loadOutBits.uop.exceptionVec(loadAccessFault) 1110 hitLoadOut.bits.uop.replayInst := s3_replayInst 1111 hitLoadOut.bits.data := s3_loadOutBits.data 1112 hitLoadOut.bits.debug.isMMIO := s3_loadOutBits.mmio 1113 hitLoadOut.bits.debug.isPerfCnt := false.B 1114 hitLoadOut.bits.debug.paddr := s3_loadOutBits.paddr 1115 hitLoadOut.bits.debug.vaddr := s3_loadOutBits.vaddr 1116 1117 when (s3_forceReplay) { 1118 hitLoadOut.bits.uop.exceptionVec := 0.U.asTypeOf(s3_loadOutBits.uop.exceptionVec.cloneType) 1119 } 1120 1121 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 1122 1123 io.lsq.loadIn.bits.uop := hitLoadOut.bits.uop 1124 1125 val s3_needRelease = s3_exception || io.lsq.loadIn.bits.replayInfo.needReplay() 1126 io.lsq.loadLoadViolationQuery.preReq := load_s1.io.out.valid 1127 io.lsq.loadLoadViolationQuery.release := s3_needRelease 1128 io.lsq.storeLoadViolationQuery.preReq := load_s1.io.out.valid 1129 io.lsq.storeLoadViolationQuery.release := s3_needRelease 1130 1131 // feedback slow 1132 io.feedbackSlow.valid := s3_loadOutValid && !s3_loadOutBits.uop.robIdx.needFlush(io.redirect) && !s3_loadOutBits.isLoadReplay 1133 io.feedbackSlow.bits.hit := !io.lsq.loadIn.bits.replayInfo.needReplay() || io.lsq.loadIn.ready 1134 io.feedbackSlow.bits.flushState := s3_loadOutBits.ptwBack 1135 io.feedbackSlow.bits.rsIdx := s3_loadOutBits.rsIdx 1136 io.feedbackSlow.bits.sourceType := RSFeedbackType.lrqFull 1137 io.feedbackSlow.bits.dataInvalidSqIdx := DontCare 1138 1139 val s3_loadWbMeta = Mux(hitLoadOut.valid, hitLoadOut.bits, io.lsq.loadOut.bits) 1140 // data from load queue refill 1141 val s3_loadDataFromLQ = io.lsq.ldRawData 1142 val s3_rdataLQ = s3_loadDataFromLQ.mergedData() 1143 val s3_rdataSelLQ = LookupTree(s3_loadDataFromLQ.addrOffset, List( 1144 "b000".U -> s3_rdataLQ(63, 0), 1145 "b001".U -> s3_rdataLQ(63, 8), 1146 "b010".U -> s3_rdataLQ(63, 16), 1147 "b011".U -> s3_rdataLQ(63, 24), 1148 "b100".U -> s3_rdataLQ(63, 32), 1149 "b101".U -> s3_rdataLQ(63, 40), 1150 "b110".U -> s3_rdataLQ(63, 48), 1151 "b111".U -> s3_rdataLQ(63, 56) 1152 )) 1153 val s3_rdataPartialLoadLQ = rdataHelper(s3_loadDataFromLQ.uop, s3_rdataSelLQ) 1154 1155 // data from dcache hit 1156 val s3_loadDataFromDcache = load_s2.io.loadDataFromDcache 1157 val s3_rdataDcache = s3_loadDataFromDcache.mergedData() 1158 val s3_rdataSelDcache = LookupTree(s3_loadDataFromDcache.addrOffset, List( 1159 "b000".U -> s3_rdataDcache(63, 0), 1160 "b001".U -> s3_rdataDcache(63, 8), 1161 "b010".U -> s3_rdataDcache(63, 16), 1162 "b011".U -> s3_rdataDcache(63, 24), 1163 "b100".U -> s3_rdataDcache(63, 32), 1164 "b101".U -> s3_rdataDcache(63, 40), 1165 "b110".U -> s3_rdataDcache(63, 48), 1166 "b111".U -> s3_rdataDcache(63, 56) 1167 )) 1168 val s3_rdataPartialLoadDcache = rdataHelper(s3_loadDataFromDcache.uop, s3_rdataSelDcache) 1169 1170 // FIXME: add 1 cycle delay ? 1171 io.loadOut.bits := s3_loadWbMeta 1172 io.loadOut.bits.data := Mux(hitLoadOut.valid, s3_rdataPartialLoadDcache, s3_rdataPartialLoadLQ) 1173 io.loadOut.valid := hitLoadOut.valid && !hitLoadOut.bits.uop.robIdx.needFlush(io.redirect) || 1174 io.lsq.loadOut.valid && !io.lsq.loadOut.bits.uop.robIdx.needFlush(io.redirect) && !hitLoadOut.valid 1175 1176 io.lsq.loadOut.ready := !hitLoadOut.valid 1177 1178 // fast load to load forward 1179 io.fastpathOut.valid := hitLoadOut.valid // for debug only 1180 io.fastpathOut.data := s3_loadDataFromDcache.mergedData() // fastpath is for ld only 1181 1182 // trigger 1183 val lastValidData = RegNext(RegEnable(io.loadOut.bits.data, io.loadOut.fire)) 1184 val hitLoadAddrTriggerHitVec = Wire(Vec(3, Bool())) 1185 val lqLoadAddrTriggerHitVec = io.lsq.trigger.lqLoadAddrTriggerHitVec 1186 (0 until 3).map{i => { 1187 val tdata2 = RegNext(io.trigger(i).tdata2) 1188 val matchType = RegNext(io.trigger(i).matchType) 1189 val tEnable = RegNext(io.trigger(i).tEnable) 1190 1191 hitLoadAddrTriggerHitVec(i) := TriggerCmp(RegNext(load_s2.io.out.bits.vaddr), tdata2, matchType, tEnable) 1192 io.trigger(i).addrHit := Mux(hitLoadOut.valid, hitLoadAddrTriggerHitVec(i), lqLoadAddrTriggerHitVec(i)) 1193 io.trigger(i).lastDataHit := TriggerCmp(lastValidData, tdata2, matchType, tEnable) 1194 }} 1195 io.lsq.trigger.hitLoadAddrTriggerHitVec := hitLoadAddrTriggerHitVec 1196 1197 // FIXME: please move this part to LoadQueueReplay 1198 io.debug_ls := DontCare 1199 // io.debug_ls.s1.isBankConflict := load_s1.io.in.fire && (!load_s1.io.dcacheKill && load_s1.io.dcacheBankConflict) 1200 // io.debug_ls.s1.isLoadToLoadForward := load_s1.io.out.valid && s1_tryPointerChasing && !cancelPointerChasing 1201 // io.debug_ls.s1.isTlbFirstMiss := io.tlb.resp.valid && io.tlb.resp.bits.miss && io.tlb.resp.bits.debug.isFirstIssue 1202 // io.debug_ls.s1.isReplayFast := io.lsq.replayFast.valid && io.lsq.replayFast.needreplay 1203 // io.debug_ls.s1_robIdx := load_s1.io.in.bits.uop.robIdx.value 1204 // // s2 1205 // io.debug_ls.s2.isDcacheFirstMiss := load_s2.io.in.fire && load_s2.io.in.bits.isFirstIssue && load_s2.io.dcacheResp.bits.miss 1206 // io.debug_ls.s2.isForwardFail := load_s2.io.in.fire && load_s2.io.s2_forward_fail 1207 // io.debug_ls.s2.isReplaySlow := io.lsq.replaySlow.valid && io.lsq.replaySlow.needreplay 1208 // io.debug_ls.s2.isLoadReplayTLBMiss := io.lsq.replaySlow.valid && !io.lsq.replaySlow.tlb_hited 1209 // io.debug_ls.s2.isLoadReplayCacheMiss := io.lsq.replaySlow.valid && !io.lsq.replaySlow.cache_hited 1210 // io.debug_ls.replayCnt := DontCare 1211 // io.debug_ls.s2_robIdx := load_s2.io.in.bits.uop.robIdx.value 1212 1213 // bug lyq: some signals in perfEvents are no longer suitable for the current MemBlock design 1214 // hardware performance counter 1215 val perfEvents = Seq( 1216 ("load_s0_in_fire ", load_s0.io.in.fire ), 1217 ("load_to_load_forward ", load_s1.io.out.valid && s1_tryPointerChasing && !cancelPointerChasing ), 1218 ("stall_dcache ", load_s0.io.out.valid && load_s0.io.out.ready && !load_s0.io.dcacheReq.ready ), 1219 ("load_s1_in_fire ", load_s1.io.in.fire ), 1220 ("load_s1_tlb_miss ", load_s1.io.in.fire && load_s1.io.dtlbResp.bits.miss ), 1221 ("load_s2_in_fire ", load_s2.io.in.fire ), 1222 ("load_s2_dcache_miss ", load_s2.io.in.fire && load_s2.io.dcacheResp.bits.miss ), 1223 ) 1224 generatePerfEvent() 1225 1226 when(io.loadOut.fire){ 1227 XSDebug("loadOut %x\n", io.loadOut.bits.uop.pc) 1228 } 1229} 1230