1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import xiangshan.ExceptionNO._ 24import xiangshan._ 25import xiangshan.backend.fu.PMPRespBundle 26import xiangshan.cache._ 27import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp} 28 29class LoadToLsqIO(implicit p: Parameters) extends XSBundle { 30 val loadIn = ValidIO(new LsPipelineBundle) 31 val ldout = Flipped(DecoupledIO(new ExuOutput)) 32 val loadDataForwarded = Output(Bool()) 33 val delayedLoadError = Output(Bool()) 34 val dcacheRequireReplay = Output(Bool()) 35 val forward = new PipeLoadForwardQueryIO 36 val loadViolationQuery = new LoadViolationQueryIO 37 val trigger = Flipped(new LqTriggerIO) 38} 39 40class LoadToLoadIO(implicit p: Parameters) extends XSBundle { 41 // load to load fast path is limited to ld (64 bit) used as vaddr src1 only 42 val data = UInt(XLEN.W) 43 val valid = Bool() 44} 45 46class LoadUnitTriggerIO(implicit p: Parameters) extends XSBundle { 47 val tdata2 = Input(UInt(64.W)) 48 val matchType = Input(UInt(2.W)) 49 val tEnable = Input(Bool()) // timing is calculated before this 50 val addrHit = Output(Bool()) 51 val lastDataHit = Output(Bool()) 52} 53 54// Load Pipeline Stage 0 55// Generate addr, use addr to query DCache and DTLB 56class LoadUnit_S0(implicit p: Parameters) extends XSModule with HasDCacheParameters{ 57 val io = IO(new Bundle() { 58 val in = Flipped(Decoupled(new ExuInput)) 59 val out = Decoupled(new LsPipelineBundle) 60 val fastpath = Input(Vec(LoadPipelineWidth, new LoadToLoadIO)) 61 val dtlbReq = DecoupledIO(new TlbReq) 62 val dcacheReq = DecoupledIO(new DCacheWordReq) 63 val rsIdx = Input(UInt(log2Up(IssQueSize).W)) 64 val isFirstIssue = Input(Bool()) 65 val loadFastMatch = Input(UInt(exuParameters.LduCnt.W)) 66 }) 67 require(LoadPipelineWidth == exuParameters.LduCnt) 68 69 val s0_uop = io.in.bits.uop 70 val imm12 = WireInit(s0_uop.ctrl.imm(11,0)) 71 72 val s0_vaddr = WireInit(io.in.bits.src(0) + SignExt(s0_uop.ctrl.imm(11,0), VAddrBits)) 73 val s0_mask = WireInit(genWmask(s0_vaddr, s0_uop.ctrl.fuOpType(1,0))) 74 75 if (EnableLoadToLoadForward) { 76 // slow vaddr from non-load insts 77 val slowpath_vaddr = io.in.bits.src(0) + SignExt(s0_uop.ctrl.imm(11,0), VAddrBits) 78 val slowpath_mask = genWmask(slowpath_vaddr, s0_uop.ctrl.fuOpType(1,0)) 79 80 // fast vaddr from load insts 81 val fastpath_vaddrs = WireInit(VecInit(List.tabulate(LoadPipelineWidth)(i => { 82 io.fastpath(i).data + SignExt(s0_uop.ctrl.imm(11,0), VAddrBits) 83 }))) 84 val fastpath_masks = WireInit(VecInit(List.tabulate(LoadPipelineWidth)(i => { 85 genWmask(fastpath_vaddrs(i), s0_uop.ctrl.fuOpType(1,0)) 86 }))) 87 val fastpath_vaddr = Mux1H(io.loadFastMatch, fastpath_vaddrs) 88 val fastpath_mask = Mux1H(io.loadFastMatch, fastpath_masks) 89 90 // select vaddr from 2 alus 91 s0_vaddr := Mux(io.loadFastMatch.orR, fastpath_vaddr, slowpath_vaddr) 92 s0_mask := Mux(io.loadFastMatch.orR, fastpath_mask, slowpath_mask) 93 XSPerfAccumulate("load_to_load_forward", io.loadFastMatch.orR && io.in.fire()) 94 } 95 96 val isSoftPrefetch = LSUOpType.isPrefetch(s0_uop.ctrl.fuOpType) 97 val isSoftPrefetchRead = s0_uop.ctrl.fuOpType === LSUOpType.prefetch_r 98 val isSoftPrefetchWrite = s0_uop.ctrl.fuOpType === LSUOpType.prefetch_w 99 100 // query DTLB 101 io.dtlbReq.valid := io.in.valid 102 io.dtlbReq.bits.vaddr := s0_vaddr 103 io.dtlbReq.bits.cmd := TlbCmd.read 104 io.dtlbReq.bits.size := LSUOpType.size(io.in.bits.uop.ctrl.fuOpType) 105 io.dtlbReq.bits.kill := DontCare 106 io.dtlbReq.bits.debug.robIdx := s0_uop.robIdx 107 io.dtlbReq.bits.debug.pc := s0_uop.cf.pc 108 io.dtlbReq.bits.debug.isFirstIssue := io.isFirstIssue 109 110 // query DCache 111 io.dcacheReq.valid := io.in.valid 112 when (isSoftPrefetchRead) { 113 io.dcacheReq.bits.cmd := MemoryOpConstants.M_PFR 114 }.elsewhen (isSoftPrefetchWrite) { 115 io.dcacheReq.bits.cmd := MemoryOpConstants.M_PFW 116 }.otherwise { 117 io.dcacheReq.bits.cmd := MemoryOpConstants.M_XRD 118 } 119 io.dcacheReq.bits.addr := s0_vaddr 120 io.dcacheReq.bits.mask := s0_mask 121 io.dcacheReq.bits.data := DontCare 122 when(isSoftPrefetch) { 123 io.dcacheReq.bits.instrtype := SOFT_PREFETCH.U 124 }.otherwise { 125 io.dcacheReq.bits.instrtype := LOAD_SOURCE.U 126 } 127 128 // TODO: update cache meta 129 io.dcacheReq.bits.id := DontCare 130 131 val addrAligned = LookupTree(s0_uop.ctrl.fuOpType(1, 0), List( 132 "b00".U -> true.B, //b 133 "b01".U -> (s0_vaddr(0) === 0.U), //h 134 "b10".U -> (s0_vaddr(1, 0) === 0.U), //w 135 "b11".U -> (s0_vaddr(2, 0) === 0.U) //d 136 )) 137 138 io.out.valid := io.in.valid && io.dcacheReq.ready 139 140 io.out.bits := DontCare 141 io.out.bits.vaddr := s0_vaddr 142 io.out.bits.mask := s0_mask 143 io.out.bits.uop := s0_uop 144 io.out.bits.uop.cf.exceptionVec(loadAddrMisaligned) := !addrAligned 145 io.out.bits.rsIdx := io.rsIdx 146 io.out.bits.isFirstIssue := io.isFirstIssue 147 io.out.bits.isSoftPrefetch := isSoftPrefetch 148 149 io.in.ready := !io.in.valid || (io.out.ready && io.dcacheReq.ready) 150 151 XSDebug(io.dcacheReq.fire(), 152 p"[DCACHE LOAD REQ] pc ${Hexadecimal(s0_uop.cf.pc)}, vaddr ${Hexadecimal(s0_vaddr)}\n" 153 ) 154 XSPerfAccumulate("in_valid", io.in.valid) 155 XSPerfAccumulate("in_fire", io.in.fire) 156 XSPerfAccumulate("in_fire_first_issue", io.in.valid && io.isFirstIssue) 157 XSPerfAccumulate("stall_out", io.out.valid && !io.out.ready && io.dcacheReq.ready) 158 XSPerfAccumulate("stall_dcache", io.out.valid && io.out.ready && !io.dcacheReq.ready) 159 XSPerfAccumulate("addr_spec_success", io.out.fire() && s0_vaddr(VAddrBits-1, 12) === io.in.bits.src(0)(VAddrBits-1, 12)) 160 XSPerfAccumulate("addr_spec_failed", io.out.fire() && s0_vaddr(VAddrBits-1, 12) =/= io.in.bits.src(0)(VAddrBits-1, 12)) 161 XSPerfAccumulate("addr_spec_success_once", io.out.fire() && s0_vaddr(VAddrBits-1, 12) === io.in.bits.src(0)(VAddrBits-1, 12) && io.isFirstIssue) 162 XSPerfAccumulate("addr_spec_failed_once", io.out.fire() && s0_vaddr(VAddrBits-1, 12) =/= io.in.bits.src(0)(VAddrBits-1, 12) && io.isFirstIssue) 163} 164 165 166// Load Pipeline Stage 1 167// TLB resp (send paddr to dcache) 168class LoadUnit_S1(implicit p: Parameters) extends XSModule { 169 val io = IO(new Bundle() { 170 val in = Flipped(Decoupled(new LsPipelineBundle)) 171 val out = Decoupled(new LsPipelineBundle) 172 val dtlbResp = Flipped(DecoupledIO(new TlbResp)) 173 val dcachePAddr = Output(UInt(PAddrBits.W)) 174 val dcacheKill = Output(Bool()) 175 val dcacheBankConflict = Input(Bool()) 176 val fullForwardFast = Output(Bool()) 177 val sbuffer = new LoadForwardQueryIO 178 val lsq = new PipeLoadForwardQueryIO 179 val loadViolationQueryReq = Decoupled(new LoadViolationQueryReq) 180 val rsFeedback = ValidIO(new RSFeedback) 181 val csrCtrl = Flipped(new CustomCSRCtrlIO) 182 val needLdVioCheckRedo = Output(Bool()) 183 }) 184 185 val s1_uop = io.in.bits.uop 186 val s1_paddr = io.dtlbResp.bits.paddr 187 // af & pf exception were modified below. 188 val s1_exception = ExceptionNO.selectByFu(io.out.bits.uop.cf.exceptionVec, lduCfg).asUInt.orR 189 val s1_tlb_miss = io.dtlbResp.bits.miss 190 val s1_mask = io.in.bits.mask 191 val s1_bank_conflict = io.dcacheBankConflict 192 193 io.out.bits := io.in.bits // forwardXX field will be updated in s1 194 195 io.dtlbResp.ready := true.B 196 197 io.dcachePAddr := s1_paddr 198 //io.dcacheKill := s1_tlb_miss || s1_exception || s1_mmio 199 io.dcacheKill := s1_tlb_miss || s1_exception 200 // load forward query datapath 201 io.sbuffer.valid := io.in.valid && !(s1_exception || s1_tlb_miss) 202 io.sbuffer.vaddr := io.in.bits.vaddr 203 io.sbuffer.paddr := s1_paddr 204 io.sbuffer.uop := s1_uop 205 io.sbuffer.sqIdx := s1_uop.sqIdx 206 io.sbuffer.mask := s1_mask 207 io.sbuffer.pc := s1_uop.cf.pc // FIXME: remove it 208 209 io.lsq.valid := io.in.valid && !(s1_exception || s1_tlb_miss) 210 io.lsq.vaddr := io.in.bits.vaddr 211 io.lsq.paddr := s1_paddr 212 io.lsq.uop := s1_uop 213 io.lsq.sqIdx := s1_uop.sqIdx 214 io.lsq.sqIdxMask := DontCare // will be overwritten by sqIdxMask pre-generated in s0 215 io.lsq.mask := s1_mask 216 io.lsq.pc := s1_uop.cf.pc // FIXME: remove it 217 218 // ld-ld violation query 219 io.loadViolationQueryReq.valid := io.in.valid && !(s1_exception || s1_tlb_miss) 220 io.loadViolationQueryReq.bits.paddr := s1_paddr 221 io.loadViolationQueryReq.bits.uop := s1_uop 222 223 // Generate forwardMaskFast to wake up insts earlier 224 val forwardMaskFast = io.lsq.forwardMaskFast.asUInt | io.sbuffer.forwardMaskFast.asUInt 225 io.fullForwardFast := (~forwardMaskFast & s1_mask) === 0.U 226 227 // Generate feedback signal caused by: 228 // * dcache bank conflict 229 // * need redo ld-ld violation check 230 val needLdVioCheckRedo = io.loadViolationQueryReq.valid && 231 !io.loadViolationQueryReq.ready && 232 RegNext(io.csrCtrl.ldld_vio_check_enable) 233 io.needLdVioCheckRedo := needLdVioCheckRedo 234 io.rsFeedback.valid := io.in.valid && (s1_bank_conflict || needLdVioCheckRedo) 235 io.rsFeedback.bits.hit := false.B // we have found s1_bank_conflict / re do ld-ld violation check 236 io.rsFeedback.bits.rsIdx := io.in.bits.rsIdx 237 io.rsFeedback.bits.flushState := io.in.bits.ptwBack 238 io.rsFeedback.bits.sourceType := Mux(s1_bank_conflict, RSFeedbackType.bankConflict, RSFeedbackType.ldVioCheckRedo) 239 io.rsFeedback.bits.dataInvalidSqIdx := DontCare 240 241 // if replay is detected in load_s1, 242 // load inst will be canceled immediately 243 io.out.valid := io.in.valid && !io.rsFeedback.valid 244 io.out.bits.paddr := s1_paddr 245 io.out.bits.tlbMiss := s1_tlb_miss 246 247 // current ori test will cause the case of ldest == 0, below will be modifeid in the future. 248 // af & pf exception were modified 249 io.out.bits.uop.cf.exceptionVec(loadPageFault) := io.dtlbResp.bits.excp.pf.ld 250 io.out.bits.uop.cf.exceptionVec(loadAccessFault) := io.dtlbResp.bits.excp.af.ld 251 252 io.out.bits.ptwBack := io.dtlbResp.bits.ptwBack 253 io.out.bits.rsIdx := io.in.bits.rsIdx 254 255 io.out.bits.isSoftPrefetch := io.in.bits.isSoftPrefetch 256 257 io.in.ready := !io.in.valid || io.out.ready 258 259 XSPerfAccumulate("in_valid", io.in.valid) 260 XSPerfAccumulate("in_fire", io.in.fire) 261 XSPerfAccumulate("in_fire_first_issue", io.in.fire && io.in.bits.isFirstIssue) 262 XSPerfAccumulate("tlb_miss", io.in.fire && s1_tlb_miss) 263 XSPerfAccumulate("tlb_miss_first_issue", io.in.fire && s1_tlb_miss && io.in.bits.isFirstIssue) 264 XSPerfAccumulate("stall_out", io.out.valid && !io.out.ready) 265} 266 267// Load Pipeline Stage 2 268// DCache resp 269class LoadUnit_S2(implicit p: Parameters) extends XSModule with HasLoadHelper { 270 val io = IO(new Bundle() { 271 val in = Flipped(Decoupled(new LsPipelineBundle)) 272 val out = Decoupled(new LsPipelineBundle) 273 val rsFeedback = ValidIO(new RSFeedback) 274 val dcacheResp = Flipped(DecoupledIO(new DCacheWordResp)) 275 val pmpResp = Flipped(new PMPRespBundle()) 276 val lsq = new LoadForwardQueryIO 277 val dataInvalidSqIdx = Input(UInt()) 278 val sbuffer = new LoadForwardQueryIO 279 val dataForwarded = Output(Bool()) 280 val dcacheRequireReplay = Output(Bool()) 281 val fullForward = Output(Bool()) 282 val fastpath = Output(new LoadToLoadIO) 283 val dcache_kill = Output(Bool()) 284 val delayedLoadError = Output(Bool()) 285 val loadViolationQueryResp = Flipped(Valid(new LoadViolationQueryResp)) 286 val csrCtrl = Flipped(new CustomCSRCtrlIO) 287 val sentFastUop = Input(Bool()) 288 val static_pm = Input(Valid(Bool())) // valid for static, bits for mmio 289 }) 290 291 val pmp = WireInit(io.pmpResp) 292 when (io.static_pm.valid) { 293 pmp.ld := false.B 294 pmp.st := false.B 295 pmp.instr := false.B 296 pmp.mmio := io.static_pm.bits 297 } 298 299 val s2_is_prefetch = io.in.bits.isSoftPrefetch 300 301 // exception that may cause load addr to be invalid / illegal 302 // 303 // if such exception happen, that inst and its exception info 304 // will be force writebacked to rob 305 val s2_exception_vec = WireInit(io.in.bits.uop.cf.exceptionVec) 306 s2_exception_vec(loadAccessFault) := io.in.bits.uop.cf.exceptionVec(loadAccessFault) || pmp.ld 307 // soft prefetch will not trigger any exception (but ecc error interrupt may be triggered) 308 when (s2_is_prefetch) { 309 s2_exception_vec := 0.U.asTypeOf(s2_exception_vec.cloneType) 310 } 311 val s2_exception = ExceptionNO.selectByFu(s2_exception_vec, lduCfg).asUInt.orR 312 313 // writeback access fault caused by ecc error / bus error 314 // 315 // * ecc data error is slow to generate, so we will not use it until load stage 3 316 // * in load stage 3, an extra signal io.load_error will be used to 317 318 // now cache ecc error will raise an access fault 319 // at the same time, error info (including error paddr) will be write to 320 // an customized CSR "CACHE_ERROR" 321 if (EnableAccurateLoadError) { 322 io.delayedLoadError := io.dcacheResp.bits.error_delayed && 323 io.csrCtrl.cache_error_enable && 324 RegNext(io.out.valid) 325 } else { 326 io.delayedLoadError := false.B 327 } 328 329 val actually_mmio = pmp.mmio 330 val s2_uop = io.in.bits.uop 331 val s2_mask = io.in.bits.mask 332 val s2_paddr = io.in.bits.paddr 333 val s2_tlb_miss = io.in.bits.tlbMiss 334 val s2_mmio = !s2_is_prefetch && actually_mmio && !s2_exception 335 val s2_cache_miss = io.dcacheResp.bits.miss 336 val s2_cache_replay = io.dcacheResp.bits.replay 337 val s2_cache_tag_error = io.dcacheResp.bits.tag_error 338 val s2_forward_fail = io.lsq.matchInvalid || io.sbuffer.matchInvalid 339 val s2_ldld_violation = io.loadViolationQueryResp.valid && 340 io.loadViolationQueryResp.bits.have_violation && 341 RegNext(io.csrCtrl.ldld_vio_check_enable) 342 val s2_data_invalid = io.lsq.dataInvalid && !s2_forward_fail && !s2_ldld_violation && !s2_exception 343 344 io.dcache_kill := pmp.ld || pmp.mmio // move pmp resp kill to outside 345 io.dcacheResp.ready := true.B 346 val dcacheShouldResp = !(s2_tlb_miss || s2_exception || s2_mmio || s2_is_prefetch) 347 assert(!(io.in.valid && (dcacheShouldResp && !io.dcacheResp.valid)), "DCache response got lost") 348 349 // merge forward result 350 // lsq has higher priority than sbuffer 351 val forwardMask = Wire(Vec(8, Bool())) 352 val forwardData = Wire(Vec(8, UInt(8.W))) 353 354 val fullForward = (~forwardMask.asUInt & s2_mask) === 0.U && !io.lsq.dataInvalid 355 io.lsq := DontCare 356 io.sbuffer := DontCare 357 io.fullForward := fullForward 358 359 // generate XLEN/8 Muxs 360 for (i <- 0 until XLEN / 8) { 361 forwardMask(i) := io.lsq.forwardMask(i) || io.sbuffer.forwardMask(i) 362 forwardData(i) := Mux(io.lsq.forwardMask(i), io.lsq.forwardData(i), io.sbuffer.forwardData(i)) 363 } 364 365 XSDebug(io.out.fire(), "[FWD LOAD RESP] pc %x fwd %x(%b) + %x(%b)\n", 366 s2_uop.cf.pc, 367 io.lsq.forwardData.asUInt, io.lsq.forwardMask.asUInt, 368 io.in.bits.forwardData.asUInt, io.in.bits.forwardMask.asUInt 369 ) 370 371 // data merge 372 val rdataVec = VecInit((0 until XLEN / 8).map(j => 373 Mux(forwardMask(j), forwardData(j), io.dcacheResp.bits.data(8*(j+1)-1, 8*j)))) 374 val rdata = rdataVec.asUInt 375 val rdataSel = LookupTree(s2_paddr(2, 0), List( 376 "b000".U -> rdata(63, 0), 377 "b001".U -> rdata(63, 8), 378 "b010".U -> rdata(63, 16), 379 "b011".U -> rdata(63, 24), 380 "b100".U -> rdata(63, 32), 381 "b101".U -> rdata(63, 40), 382 "b110".U -> rdata(63, 48), 383 "b111".U -> rdata(63, 56) 384 )) 385 val rdataPartialLoad = rdataHelper(s2_uop, rdataSel) 386 387 io.out.valid := io.in.valid && !s2_tlb_miss && !s2_data_invalid 388 // Inst will be canceled in store queue / lsq, 389 // so we do not need to care about flush in load / store unit's out.valid 390 io.out.bits := io.in.bits 391 io.out.bits.data := rdataPartialLoad 392 // when exception occurs, set it to not miss and let it write back to rob (via int port) 393 if (EnableFastForward) { 394 io.out.bits.miss := s2_cache_miss && 395 !s2_exception && 396 !s2_forward_fail && 397 !s2_ldld_violation && 398 !fullForward && 399 !s2_is_prefetch 400 } else { 401 io.out.bits.miss := s2_cache_miss && 402 !s2_exception && 403 !s2_forward_fail && 404 !s2_ldld_violation && 405 !s2_is_prefetch 406 } 407 io.out.bits.uop.ctrl.fpWen := io.in.bits.uop.ctrl.fpWen && !s2_exception 408 // if forward fail, replay this inst from fetch 409 val forwardFailReplay = s2_forward_fail && !s2_mmio && !s2_is_prefetch && !s2_tlb_miss 410 // if ld-ld violation is detected, replay from this inst from fetch 411 val ldldVioReplay = s2_ldld_violation && !s2_mmio && !s2_is_prefetch && !s2_tlb_miss 412 val s2_need_replay_from_fetch = (s2_forward_fail || s2_ldld_violation) && !s2_mmio && !s2_is_prefetch && !s2_tlb_miss 413 io.out.bits.uop.ctrl.replayInst := s2_need_replay_from_fetch 414 io.out.bits.mmio := s2_mmio 415 io.out.bits.uop.ctrl.flushPipe := s2_mmio && io.sentFastUop 416 io.out.bits.uop.cf.exceptionVec := s2_exception_vec // cache error not included 417 418 // For timing reasons, sometimes we can not let 419 // io.out.bits.miss := s2_cache_miss && !s2_exception && !fullForward 420 // We use io.dataForwarded instead. It means: 421 // 1. Forward logic have prepared all data needed, 422 // and dcache query is no longer needed. 423 // 2. ... or data cache tag error is detected, this kind of inst 424 // will not update miss queue. That is to say, if miss, that inst 425 // may not be refilled 426 // Such inst will be writebacked from load queue. 427 io.dataForwarded := s2_cache_miss && !s2_exception && !s2_forward_fail && 428 (fullForward || io.csrCtrl.cache_error_enable && s2_cache_tag_error) 429 // io.out.bits.forwardX will be send to lq 430 io.out.bits.forwardMask := forwardMask 431 // data retbrived from dcache is also included in io.out.bits.forwardData 432 io.out.bits.forwardData := rdataVec 433 434 io.in.ready := io.out.ready || !io.in.valid 435 436 // feedback tlb result to RS 437 io.rsFeedback.valid := io.in.valid 438 val s2_need_replay_from_rs = Wire(Bool()) 439 if (EnableFastForward) { 440 s2_need_replay_from_rs := 441 s2_tlb_miss || // replay if dtlb miss 442 s2_cache_replay && !s2_is_prefetch && !s2_forward_fail && !s2_ldld_violation && !s2_mmio && !s2_exception && !fullForward || // replay if dcache miss queue full / busy 443 s2_data_invalid && !s2_is_prefetch && !s2_forward_fail && !s2_ldld_violation // replay if store to load forward data is not ready 444 } else { 445 // Note that if all parts of data are available in sq / sbuffer, replay required by dcache will not be scheduled 446 s2_need_replay_from_rs := 447 s2_tlb_miss || // replay if dtlb miss 448 s2_cache_replay && !s2_is_prefetch && !s2_forward_fail && !s2_ldld_violation && !s2_mmio && !s2_exception && !io.dataForwarded || // replay if dcache miss queue full / busy 449 s2_data_invalid && !s2_is_prefetch && !s2_forward_fail && !s2_ldld_violation // replay if store to load forward data is not ready 450 } 451 assert(!RegNext(io.in.valid && s2_need_replay_from_rs && s2_need_replay_from_fetch)) 452 io.rsFeedback.bits.hit := !s2_need_replay_from_rs 453 io.rsFeedback.bits.rsIdx := io.in.bits.rsIdx 454 io.rsFeedback.bits.flushState := io.in.bits.ptwBack 455 // feedback source priority: tlbMiss > dataInvalid > mshrFull 456 // general case priority: tlbMiss > exception (include forward_fail / ldld_violation) > mmio > dataInvalid > mshrFull > normal miss / hit 457 io.rsFeedback.bits.sourceType := Mux(s2_tlb_miss, RSFeedbackType.tlbMiss, 458 Mux(s2_data_invalid, 459 RSFeedbackType.dataInvalid, 460 RSFeedbackType.mshrFull 461 ) 462 ) 463 io.rsFeedback.bits.dataInvalidSqIdx.value := io.dataInvalidSqIdx 464 io.rsFeedback.bits.dataInvalidSqIdx.flag := DontCare 465 466 // s2_cache_replay is quite slow to generate, send it separately to LQ 467 if (EnableFastForward) { 468 io.dcacheRequireReplay := s2_cache_replay && !fullForward 469 } else { 470 io.dcacheRequireReplay := s2_cache_replay && 471 !io.rsFeedback.bits.hit && 472 !io.dataForwarded && 473 !s2_is_prefetch && 474 io.out.bits.miss 475 } 476 477 // fast load to load forward 478 io.fastpath.valid := RegNext(io.out.valid) // for debug only 479 io.fastpath.data := RegNext(io.out.bits.data) 480 481 482 XSDebug(io.out.fire(), "[DCACHE LOAD RESP] pc %x rdata %x <- D$ %x + fwd %x(%b)\n", 483 s2_uop.cf.pc, rdataPartialLoad, io.dcacheResp.bits.data, 484 forwardData.asUInt, forwardMask.asUInt 485 ) 486 487 XSPerfAccumulate("in_valid", io.in.valid) 488 XSPerfAccumulate("in_fire", io.in.fire) 489 XSPerfAccumulate("in_fire_first_issue", io.in.fire && io.in.bits.isFirstIssue) 490 XSPerfAccumulate("dcache_miss", io.in.fire && s2_cache_miss) 491 XSPerfAccumulate("dcache_miss_first_issue", io.in.fire && s2_cache_miss && io.in.bits.isFirstIssue) 492 XSPerfAccumulate("full_forward", io.in.valid && fullForward) 493 XSPerfAccumulate("dcache_miss_full_forward", io.in.valid && s2_cache_miss && fullForward) 494 XSPerfAccumulate("replay", io.rsFeedback.valid && !io.rsFeedback.bits.hit) 495 XSPerfAccumulate("replay_tlb_miss", io.rsFeedback.valid && !io.rsFeedback.bits.hit && s2_tlb_miss) 496 XSPerfAccumulate("replay_cache", io.rsFeedback.valid && !io.rsFeedback.bits.hit && !s2_tlb_miss && s2_cache_replay) 497 XSPerfAccumulate("stall_out", io.out.valid && !io.out.ready) 498 XSPerfAccumulate("replay_from_fetch_forward", io.out.valid && forwardFailReplay) 499 XSPerfAccumulate("replay_from_fetch_load_vio", io.out.valid && ldldVioReplay) 500} 501 502class LoadUnit(implicit p: Parameters) extends XSModule 503 with HasLoadHelper 504 with HasPerfEvents 505 with HasDCacheParameters 506{ 507 val io = IO(new Bundle() { 508 val ldin = Flipped(Decoupled(new ExuInput)) 509 val ldout = Decoupled(new ExuOutput) 510 val redirect = Flipped(ValidIO(new Redirect)) 511 val feedbackSlow = ValidIO(new RSFeedback) 512 val feedbackFast = ValidIO(new RSFeedback) 513 val rsIdx = Input(UInt(log2Up(IssQueSize).W)) 514 val isFirstIssue = Input(Bool()) 515 val dcache = new DCacheLoadIO 516 val sbuffer = new LoadForwardQueryIO 517 val lsq = new LoadToLsqIO 518 val refill = Flipped(ValidIO(new Refill)) 519 val fastUop = ValidIO(new MicroOp) // early wakeup signal generated in load_s1, send to RS in load_s2 520 val trigger = Vec(3, new LoadUnitTriggerIO) 521 522 val tlb = new TlbRequestIO 523 val pmp = Flipped(new PMPRespBundle()) // arrive same to tlb now 524 525 val fastpathOut = Output(new LoadToLoadIO) 526 val fastpathIn = Input(Vec(LoadPipelineWidth, new LoadToLoadIO)) 527 val loadFastMatch = Input(UInt(exuParameters.LduCnt.W)) 528 529 val delayedLoadError = Output(Bool()) // load ecc error 530 // Note that io.delayedLoadError and io.lsq.delayedLoadError is different 531 532 val csrCtrl = Flipped(new CustomCSRCtrlIO) 533 }) 534 535 val load_s0 = Module(new LoadUnit_S0) 536 val load_s1 = Module(new LoadUnit_S1) 537 val load_s2 = Module(new LoadUnit_S2) 538 539 load_s0.io.in <> io.ldin 540 load_s0.io.dtlbReq <> io.tlb.req 541 load_s0.io.dcacheReq <> io.dcache.req 542 load_s0.io.rsIdx := io.rsIdx 543 load_s0.io.isFirstIssue := io.isFirstIssue 544 load_s0.io.fastpath := io.fastpathIn 545 load_s0.io.loadFastMatch := io.loadFastMatch 546 547 PipelineConnect(load_s0.io.out, load_s1.io.in, true.B, load_s0.io.out.bits.uop.robIdx.needFlush(io.redirect)) 548 549 load_s1.io.dtlbResp <> io.tlb.resp 550 io.dcache.s1_paddr <> load_s1.io.dcachePAddr 551 io.dcache.s1_kill <> load_s1.io.dcacheKill 552 load_s1.io.sbuffer <> io.sbuffer 553 load_s1.io.lsq <> io.lsq.forward 554 load_s1.io.loadViolationQueryReq <> io.lsq.loadViolationQuery.req 555 load_s1.io.dcacheBankConflict <> io.dcache.s1_bank_conflict 556 load_s1.io.csrCtrl <> io.csrCtrl 557 558 PipelineConnect(load_s1.io.out, load_s2.io.in, true.B, load_s1.io.out.bits.uop.robIdx.needFlush(io.redirect)) 559 560 io.dcache.s2_kill := load_s2.io.dcache_kill // to kill mmio resp which are redirected 561 load_s2.io.dcacheResp <> io.dcache.resp 562 load_s2.io.pmpResp <> io.pmp 563 load_s2.io.static_pm := RegNext(io.tlb.resp.bits.static_pm) 564 load_s2.io.lsq.forwardData <> io.lsq.forward.forwardData 565 load_s2.io.lsq.forwardMask <> io.lsq.forward.forwardMask 566 load_s2.io.lsq.forwardMaskFast <> io.lsq.forward.forwardMaskFast // should not be used in load_s2 567 load_s2.io.lsq.dataInvalid <> io.lsq.forward.dataInvalid 568 load_s2.io.lsq.matchInvalid <> io.lsq.forward.matchInvalid 569 load_s2.io.sbuffer.forwardData <> io.sbuffer.forwardData 570 load_s2.io.sbuffer.forwardMask <> io.sbuffer.forwardMask 571 load_s2.io.sbuffer.forwardMaskFast <> io.sbuffer.forwardMaskFast // should not be used in load_s2 572 load_s2.io.sbuffer.dataInvalid <> io.sbuffer.dataInvalid // always false 573 load_s2.io.sbuffer.matchInvalid <> io.sbuffer.matchInvalid 574 load_s2.io.dataForwarded <> io.lsq.loadDataForwarded 575 load_s2.io.fastpath <> io.fastpathOut 576 load_s2.io.dataInvalidSqIdx := io.lsq.forward.dataInvalidSqIdx // provide dataInvalidSqIdx to make wakeup faster 577 load_s2.io.loadViolationQueryResp <> io.lsq.loadViolationQuery.resp 578 load_s2.io.csrCtrl <> io.csrCtrl 579 load_s2.io.sentFastUop := io.fastUop.valid 580 581 // actually load s3 582 io.lsq.dcacheRequireReplay := load_s2.io.dcacheRequireReplay 583 io.lsq.delayedLoadError := load_s2.io.delayedLoadError 584 585 // feedback tlb miss / dcache miss queue full 586 io.feedbackSlow.valid := RegNext(load_s2.io.rsFeedback.valid && !load_s2.io.out.bits.uop.robIdx.needFlush(io.redirect)) 587 io.feedbackSlow.bits := RegNext(load_s2.io.rsFeedback.bits) 588 val s3_replay_for_mshrfull = RegNext(!load_s2.io.rsFeedback.bits.hit && load_s2.io.rsFeedback.bits.sourceType === RSFeedbackType.mshrFull) 589 val s3_refill_hit_load_paddr = refill_addr_hit(RegNext(load_s2.io.out.bits.paddr), io.refill.bits.addr) 590 // update replay request 591 io.feedbackSlow.bits.hit := RegNext(load_s2.io.rsFeedback.bits).hit || 592 s3_refill_hit_load_paddr && s3_replay_for_mshrfull 593 594 // feedback bank conflict / ld-vio check struct hazard to rs 595 io.feedbackFast.bits := RegNext(load_s1.io.rsFeedback.bits) 596 io.feedbackFast.valid := RegNext(load_s1.io.rsFeedback.valid && !load_s1.io.out.bits.uop.robIdx.needFlush(io.redirect)) 597 // If replay is reported at load_s1, inst will be canceled (will not enter load_s2), 598 // in that case: 599 // * replay should not be reported twice 600 assert(!(RegNext(io.feedbackFast.valid) && io.feedbackSlow.valid)) 601 // * io.fastUop.valid should not be reported 602 assert(!RegNext(io.feedbackFast.valid && io.fastUop.valid)) 603 604 // pre-calcuate sqIdx mask in s0, then send it to lsq in s1 for forwarding 605 val sqIdxMaskReg = RegNext(UIntToMask(load_s0.io.in.bits.uop.sqIdx.value, StoreQueueSize)) 606 io.lsq.forward.sqIdxMask := sqIdxMaskReg 607 608 // // use s2_hit_way to select data received in s1 609 // load_s2.io.dcacheResp.bits.data := Mux1H(RegNext(io.dcache.s1_hit_way), RegNext(io.dcache.s1_data)) 610 // assert(load_s2.io.dcacheResp.bits.data === io.dcache.resp.bits.data) 611 612 // now io.fastUop.valid is sent to RS in load_s2 613 io.fastUop.valid := RegNext( 614 io.dcache.s1_hit_way.orR && // dcache hit 615 !io.dcache.s1_disable_fast_wakeup && // load fast wakeup should be disabled when dcache data read is not ready 616 load_s1.io.in.valid && // valid laod request 617 !load_s1.io.dtlbResp.bits.fast_miss && // not mmio or tlb miss, pf / af not included here 618 !io.lsq.forward.dataInvalidFast && // forward failed 619 !load_s1.io.needLdVioCheckRedo // load-load violation check: load paddr cam struct hazard 620 ) && !RegNext(load_s1.io.out.bits.uop.robIdx.needFlush(io.redirect)) 621 io.fastUop.bits := RegNext(load_s1.io.out.bits.uop) 622 623 XSDebug(load_s0.io.out.valid, 624 p"S0: pc ${Hexadecimal(load_s0.io.out.bits.uop.cf.pc)}, lId ${Hexadecimal(load_s0.io.out.bits.uop.lqIdx.asUInt)}, " + 625 p"vaddr ${Hexadecimal(load_s0.io.out.bits.vaddr)}, mask ${Hexadecimal(load_s0.io.out.bits.mask)}\n") 626 XSDebug(load_s1.io.out.valid, 627 p"S1: pc ${Hexadecimal(load_s1.io.out.bits.uop.cf.pc)}, lId ${Hexadecimal(load_s1.io.out.bits.uop.lqIdx.asUInt)}, tlb_miss ${io.tlb.resp.bits.miss}, " + 628 p"paddr ${Hexadecimal(load_s1.io.out.bits.paddr)}, mmio ${load_s1.io.out.bits.mmio}\n") 629 630 // writeback to LSQ 631 // Current dcache use MSHR 632 // Load queue will be updated at s2 for both hit/miss int/fp load 633 io.lsq.loadIn.valid := load_s2.io.out.valid 634 io.lsq.loadIn.bits := load_s2.io.out.bits 635 636 // write to rob and writeback bus 637 val s2_wb_valid = load_s2.io.out.valid && !load_s2.io.out.bits.miss && !load_s2.io.out.bits.mmio 638 639 // Int load, if hit, will be writebacked at s2 640 val hitLoadOut = Wire(Valid(new ExuOutput)) 641 hitLoadOut.valid := s2_wb_valid 642 hitLoadOut.bits.uop := load_s2.io.out.bits.uop 643 hitLoadOut.bits.data := load_s2.io.out.bits.data 644 hitLoadOut.bits.redirectValid := false.B 645 hitLoadOut.bits.redirect := DontCare 646 hitLoadOut.bits.debug.isMMIO := load_s2.io.out.bits.mmio 647 hitLoadOut.bits.debug.isPerfCnt := false.B 648 hitLoadOut.bits.debug.paddr := load_s2.io.out.bits.paddr 649 hitLoadOut.bits.debug.vaddr := load_s2.io.out.bits.vaddr 650 hitLoadOut.bits.fflags := DontCare 651 652 load_s2.io.out.ready := true.B 653 654 val load_wb_reg = RegNext(Mux(hitLoadOut.valid, hitLoadOut.bits, io.lsq.ldout.bits)) 655 io.ldout.bits := load_wb_reg 656 io.ldout.valid := RegNext(hitLoadOut.valid) && !RegNext(load_s2.io.out.bits.uop.robIdx.needFlush(io.redirect)) || 657 RegNext(io.lsq.ldout.valid) && !RegNext(io.lsq.ldout.bits.uop.robIdx.needFlush(io.redirect)) && !RegNext(hitLoadOut.valid) 658 659 // io.ldout.bits.uop.cf.exceptionVec(loadAccessFault) := load_wb_reg.uop.cf.exceptionVec(loadAccessFault) || 660 // hitLoadOut.valid && load_s2.io.delayedLoadError 661 662 // io.delayedLoadError := false.B 663 664 io.delayedLoadError := hitLoadOut.valid && load_s2.io.delayedLoadError 665 666 io.lsq.ldout.ready := !hitLoadOut.valid 667 668 when(io.feedbackSlow.valid && !io.feedbackSlow.bits.hit){ 669 // when need replay from rs, inst should not be writebacked to rob 670 assert(RegNext(!hitLoadOut.valid)) 671 // when need replay from rs 672 // * inst should not be writebacked to lq, or 673 // * lq state will be updated in load_s3 (next cycle) 674 assert(RegNext(!io.lsq.loadIn.valid) || RegNext(load_s2.io.dcacheRequireReplay)) 675 } 676 677 val lastValidData = RegEnable(io.ldout.bits.data, io.ldout.fire()) 678 val hitLoadAddrTriggerHitVec = Wire(Vec(3, Bool())) 679 val lqLoadAddrTriggerHitVec = io.lsq.trigger.lqLoadAddrTriggerHitVec 680 (0 until 3).map{i => { 681 val tdata2 = io.trigger(i).tdata2 682 val matchType = io.trigger(i).matchType 683 val tEnable = io.trigger(i).tEnable 684 685 hitLoadAddrTriggerHitVec(i) := TriggerCmp(load_s2.io.out.bits.vaddr, tdata2, matchType, tEnable) 686 io.trigger(i).addrHit := Mux(hitLoadOut.valid, hitLoadAddrTriggerHitVec(i), lqLoadAddrTriggerHitVec(i)) 687 io.trigger(i).lastDataHit := TriggerCmp(lastValidData, tdata2, matchType, tEnable) 688 }} 689 io.lsq.trigger.hitLoadAddrTriggerHitVec := hitLoadAddrTriggerHitVec 690 691 val perfEvents = Seq( 692 ("load_s0_in_fire ", load_s0.io.in.fire() ), 693 ("stall_dcache ", load_s0.io.out.valid && load_s0.io.out.ready && !load_s0.io.dcacheReq.ready ), 694 ("load_s1_in_fire ", load_s1.io.in.fire ), 695 ("load_s1_tlb_miss ", load_s1.io.in.fire && load_s1.io.dtlbResp.bits.miss ), 696 ("load_s2_in_fire ", load_s2.io.in.fire ), 697 ("load_s2_dcache_miss ", load_s2.io.in.fire && load_s2.io.dcacheResp.bits.miss ), 698 ("load_s2_replay ", load_s2.io.rsFeedback.valid && !load_s2.io.rsFeedback.bits.hit ), 699 ("load_s2_replay_tlb_miss ", load_s2.io.rsFeedback.valid && !load_s2.io.rsFeedback.bits.hit && load_s2.io.in.bits.tlbMiss ), 700 ("load_s2_replay_cache ", load_s2.io.rsFeedback.valid && !load_s2.io.rsFeedback.bits.hit && !load_s2.io.in.bits.tlbMiss && load_s2.io.dcacheResp.bits.miss), 701 ) 702 generatePerfEvent() 703 704 // Will cause timing problem: 705 // ("load_to_load_forward ", load_s0.io.loadFastMatch.orR && load_s0.io.in.fire()), 706 707 when(io.ldout.fire()){ 708 XSDebug("ldout %x\n", io.ldout.bits.uop.cf.pc) 709 } 710} 711