1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan._ 25import xiangshan.backend.Bundles.{DynInst, MemExuOutput} 26import xiangshan.cache._ 27import xiangshan.cache.{DCacheLineIO, DCacheWordIO, MemoryOpConstants} 28import xiangshan.cache.mmu.TlbRequestIO 29import xiangshan.mem._ 30import xiangshan.backend._ 31import xiangshan.backend.rob.RobLsqIO 32 33class ExceptionAddrIO(implicit p: Parameters) extends XSBundle { 34 val isStore = Input(Bool()) 35 val vaddr = Output(UInt(VAddrBits.W)) 36} 37 38class FwdEntry extends Bundle { 39 val validFast = Bool() // validFast is generated the same cycle with query 40 val valid = Bool() // valid is generated 1 cycle after query request 41 val data = UInt(8.W) // data is generated 1 cycle after query request 42} 43 44// inflight miss block reqs 45class InflightBlockInfo(implicit p: Parameters) extends XSBundle { 46 val block_addr = UInt(PAddrBits.W) 47 val valid = Bool() 48} 49 50class LsqEnqIO(implicit p: Parameters) extends MemBlockBundle { 51 val canAccept = Output(Bool()) 52 val needAlloc = Vec(LSQEnqWidth, Input(UInt(2.W))) 53 val req = Vec(LSQEnqWidth, Flipped(ValidIO(new DynInst))) 54 val resp = Vec(LSQEnqWidth, Output(new LSIdx)) 55} 56 57// Load / Store Queue Wrapper for XiangShan Out of Order LSU 58class LsqWrapper(implicit p: Parameters) extends XSModule with HasDCacheParameters with HasPerfEvents { 59 val io = IO(new Bundle() { 60 val hartId = Input(UInt(8.W)) 61 val brqRedirect = Flipped(ValidIO(new Redirect)) 62 val enq = new LsqEnqIO 63 val ldu = new Bundle() { 64 val stld_nuke_query = Vec(LoadPipelineWidth, Flipped(new LoadNukeQueryIO)) // from load_s2 65 val ldld_nuke_query = Vec(LoadPipelineWidth, Flipped(new LoadNukeQueryIO)) // from load_s2 66 val ldin = Vec(LoadPipelineWidth, Flipped(Decoupled(new LqWriteBundle))) // from load_s3 67 } 68 val sta = new Bundle() { 69 val storeMaskIn = Vec(StorePipelineWidth, Flipped(Valid(new StoreMaskBundle))) // from store_s0, store mask, send to sq from rs 70 val storeAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // from store_s1 71 val storeAddrInRe = Vec(StorePipelineWidth, Input(new LsPipelineBundle())) // from store_s2 72 val vecStoreAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) 73 } 74 val std = new Bundle() { 75 val storeDataIn = Vec(StorePipelineWidth, Flipped(Valid(new MemExuOutput))) // from store_s0, store data, send to sq from rs 76 } 77 val ldout = Vec(LoadPipelineWidth, DecoupledIO(new MemExuOutput)) 78 val ld_raw_data = Vec(LoadPipelineWidth, Output(new LoadDataFromLQBundle)) 79 val replay = Vec(LoadPipelineWidth, Decoupled(new LsPipelineBundle)) 80 val sbuffer = Vec(EnsbufferWidth, Decoupled(new DCacheWordReqWithVaddrAndPfFlag)) 81 val forward = Vec(LoadPipelineWidth, Flipped(new PipeLoadForwardQueryIO)) 82 val rob = Flipped(new RobLsqIO) 83 val rollback = Output(Valid(new Redirect)) 84 val release = Flipped(Valid(new Release)) 85 val refill = Flipped(Valid(new Refill)) 86 val tl_d_channel = Input(new DcacheToLduForwardIO) 87 val uncacheOutstanding = Input(Bool()) 88 val uncache = new UncacheWordIO 89 val mmioStout = DecoupledIO(new MemExuOutput) // writeback uncached store 90 val sqEmpty = Output(Bool()) 91 val lq_rep_full = Output(Bool()) 92 val sqFull = Output(Bool()) 93 val lqFull = Output(Bool()) 94 val sqCancelCnt = Output(UInt(log2Up(StoreQueueSize+1).W)) 95 val lqCancelCnt = Output(UInt(log2Up(VirtualLoadQueueSize+1).W)) 96 val lqDeq = Output(UInt(log2Up(CommitWidth + 1).W)) 97 val sqDeq = Output(UInt(log2Ceil(EnsbufferWidth + 1).W)) 98 val lqCanAccept = Output(Bool()) 99 val sqCanAccept = Output(Bool()) 100 val lqDeqPtr = Output(new LqPtr) 101 val sqDeqPtr = Output(new SqPtr) 102 val exceptionAddr = new ExceptionAddrIO 103 val trigger = Vec(LoadPipelineWidth, new LqTriggerIO) 104 val issuePtrExt = Output(new SqPtr) 105 val l2_hint = Input(Valid(new L2ToL1Hint())) 106 val force_write = Output(Bool()) 107 val lqEmpty = Output(Bool()) 108 109 // vector 110 val vecWriteback = Flipped(ValidIO(new MemExuOutput(isVector = true))) 111 val vecStoreRetire = Flipped(ValidIO(new SqPtr)) 112 113 // top-down 114 val debugTopDown = new LoadQueueTopDownIO 115 }) 116 117 val loadQueue = Module(new LoadQueue) 118 val storeQueue = Module(new StoreQueue) 119 120 storeQueue.io.hartId := io.hartId 121 storeQueue.io.uncacheOutstanding := io.uncacheOutstanding 122 123 124 dontTouch(loadQueue.io.tlbReplayDelayCycleCtrl) 125 // Todo: imm 126 val tlbReplayDelayCycleCtrl = WireInit(VecInit(Seq(14.U(ReSelectLen.W), 0.U(ReSelectLen.W), 125.U(ReSelectLen.W), 0.U(ReSelectLen.W)))) 127 loadQueue.io.tlbReplayDelayCycleCtrl := tlbReplayDelayCycleCtrl 128 129 // io.enq logic 130 // LSQ: send out canAccept when both load queue and store queue are ready 131 // Dispatch: send instructions to LSQ only when they are ready 132 io.enq.canAccept := loadQueue.io.enq.canAccept && storeQueue.io.enq.canAccept 133 io.lqCanAccept := loadQueue.io.enq.canAccept 134 io.sqCanAccept := storeQueue.io.enq.canAccept 135 loadQueue.io.enq.sqCanAccept := storeQueue.io.enq.canAccept 136 storeQueue.io.enq.lqCanAccept := loadQueue.io.enq.canAccept 137 io.lqDeqPtr := loadQueue.io.lqDeqPtr 138 io.sqDeqPtr := storeQueue.io.sqDeqPtr 139 for (i <- io.enq.req.indices) { 140 loadQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i)(0) 141 loadQueue.io.enq.req(i).valid := io.enq.needAlloc(i)(0) && io.enq.req(i).valid 142 loadQueue.io.enq.req(i).bits := io.enq.req(i).bits 143 loadQueue.io.enq.req(i).bits.sqIdx := storeQueue.io.enq.resp(i) 144 145 storeQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i)(1) 146 storeQueue.io.enq.req(i).valid := io.enq.needAlloc(i)(1) && io.enq.req(i).valid 147 storeQueue.io.enq.req(i).bits := io.enq.req(i).bits 148 storeQueue.io.enq.req(i).bits := io.enq.req(i).bits 149 storeQueue.io.enq.req(i).bits.lqIdx := loadQueue.io.enq.resp(i) 150 151 io.enq.resp(i).lqIdx := loadQueue.io.enq.resp(i) 152 io.enq.resp(i).sqIdx := storeQueue.io.enq.resp(i) 153 } 154 155 // store queue wiring 156 storeQueue.io.brqRedirect <> io.brqRedirect 157 storeQueue.io.storeAddrIn <> io.sta.storeAddrIn // from store_s1 158 storeQueue.io.storeAddrInRe <> io.sta.storeAddrInRe // from store_s2 159 storeQueue.io.storeDataIn <> io.std.storeDataIn // from store_s0 160 storeQueue.io.storeMaskIn <> io.sta.storeMaskIn // from store_s0 161 storeQueue.io.sbuffer <> io.sbuffer 162 storeQueue.io.mmioStout <> io.mmioStout 163 storeQueue.io.rob <> io.rob 164 storeQueue.io.exceptionAddr.isStore := DontCare 165 storeQueue.io.sqCancelCnt <> io.sqCancelCnt 166 storeQueue.io.sqDeq <> io.sqDeq 167 storeQueue.io.sqEmpty <> io.sqEmpty 168 storeQueue.io.sqFull <> io.sqFull 169 storeQueue.io.forward <> io.forward // overlap forwardMask & forwardData, DO NOT CHANGE SEQUENCE 170 storeQueue.io.force_write <> io.force_write 171 storeQueue.io.vecStoreRetire <> io.vecStoreRetire 172 173 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 174 175 // load queue wiring 176 loadQueue.io.redirect <> io.brqRedirect 177 loadQueue.io.ldu <> io.ldu 178 loadQueue.io.ldout <> io.ldout 179 loadQueue.io.ld_raw_data <> io.ld_raw_data 180 loadQueue.io.rob <> io.rob 181 loadQueue.io.rollback <> io.rollback 182 loadQueue.io.replay <> io.replay 183 loadQueue.io.refill <> io.refill 184 loadQueue.io.tl_d_channel <> io.tl_d_channel 185 loadQueue.io.release <> io.release 186 loadQueue.io.trigger <> io.trigger 187 loadQueue.io.exceptionAddr.isStore := DontCare 188 loadQueue.io.lqCancelCnt <> io.lqCancelCnt 189 loadQueue.io.sq.stAddrReadySqPtr <> storeQueue.io.stAddrReadySqPtr 190 loadQueue.io.sq.stAddrReadyVec <> storeQueue.io.stAddrReadyVec 191 loadQueue.io.sq.stDataReadySqPtr <> storeQueue.io.stDataReadySqPtr 192 loadQueue.io.sq.stDataReadyVec <> storeQueue.io.stDataReadyVec 193 loadQueue.io.sq.stIssuePtr <> storeQueue.io.stIssuePtr 194 loadQueue.io.sq.sqEmpty <> storeQueue.io.sqEmpty 195 loadQueue.io.sta.storeAddrIn <> io.sta.storeAddrIn // store_s1 196 loadQueue.io.sta.vecStoreAddrIn <> io.sta.vecStoreAddrIn // store_s1 197 loadQueue.io.std.storeDataIn <> io.std.storeDataIn // store_s0 198 loadQueue.io.lqFull <> io.lqFull 199 loadQueue.io.lq_rep_full <> io.lq_rep_full 200 loadQueue.io.lqDeq <> io.lqDeq 201 loadQueue.io.l2_hint <> io.l2_hint 202 loadQueue.io.lqEmpty <> io.lqEmpty 203 loadQueue.io.vecWriteback <> io.vecWriteback 204 205 // rob commits for lsq is delayed for two cycles, which causes the delayed update for deqPtr in lq/sq 206 // s0: commit 207 // s1: exception find 208 // s2: exception triggered 209 // s3: ptr updated & new address 210 // address will be used at the next cycle after exception is triggered 211 io.exceptionAddr.vaddr := Mux(RegNext(io.exceptionAddr.isStore), storeQueue.io.exceptionAddr.vaddr, loadQueue.io.exceptionAddr.vaddr) 212 io.issuePtrExt := storeQueue.io.stAddrReadySqPtr 213 214 // naive uncache arbiter 215 val s_idle :: s_load :: s_store :: Nil = Enum(3) 216 val pendingstate = RegInit(s_idle) 217 218 switch(pendingstate){ 219 is(s_idle){ 220 when(io.uncache.req.fire && !io.uncacheOutstanding){ 221 pendingstate := Mux(loadQueue.io.uncache.req.valid, s_load, 222 Mux(io.uncacheOutstanding, s_idle, s_store)) 223 } 224 } 225 is(s_load){ 226 when(io.uncache.resp.fire){ 227 pendingstate := s_idle 228 } 229 } 230 is(s_store){ 231 when(io.uncache.resp.fire){ 232 pendingstate := s_idle 233 } 234 } 235 } 236 237 loadQueue.io.uncache := DontCare 238 storeQueue.io.uncache := DontCare 239 loadQueue.io.uncache.req.ready := false.B 240 storeQueue.io.uncache.req.ready := false.B 241 loadQueue.io.uncache.resp.valid := false.B 242 storeQueue.io.uncache.resp.valid := false.B 243 when(loadQueue.io.uncache.req.valid){ 244 io.uncache.req <> loadQueue.io.uncache.req 245 }.otherwise{ 246 io.uncache.req <> storeQueue.io.uncache.req 247 } 248 when (io.uncacheOutstanding) { 249 io.uncache.resp <> loadQueue.io.uncache.resp 250 } .otherwise { 251 when(pendingstate === s_load){ 252 io.uncache.resp <> loadQueue.io.uncache.resp 253 }.otherwise{ 254 io.uncache.resp <> storeQueue.io.uncache.resp 255 } 256 } 257 258 loadQueue.io.debugTopDown <> io.debugTopDown 259 260 assert(!(loadQueue.io.uncache.req.valid && storeQueue.io.uncache.req.valid)) 261 assert(!(loadQueue.io.uncache.resp.valid && storeQueue.io.uncache.resp.valid)) 262 when (!io.uncacheOutstanding) { 263 assert(!((loadQueue.io.uncache.resp.valid || storeQueue.io.uncache.resp.valid) && pendingstate === s_idle)) 264 } 265 266 267 val perfEvents = Seq(loadQueue, storeQueue).flatMap(_.getPerfEvents) 268 generatePerfEvent() 269} 270 271class LsqEnqCtrl(implicit p: Parameters) extends XSModule { 272 val io = IO(new Bundle { 273 val redirect = Flipped(ValidIO(new Redirect)) 274 // to dispatch 275 val enq = new LsqEnqIO 276 // from `memBlock.io.lqDeq 277 val lcommit = Input(UInt(log2Up(CommitWidth + 1).W)) 278 // from `memBlock.io.sqDeq` 279 val scommit = Input(UInt(log2Ceil(EnsbufferWidth + 1).W)) 280 // from/tp lsq 281 val lqCancelCnt = Input(UInt(log2Up(VirtualLoadQueueSize + 1).W)) 282 val sqCancelCnt = Input(UInt(log2Up(StoreQueueSize + 1).W)) 283 val enqLsq = Flipped(new LsqEnqIO) 284 }) 285 286 val lqPtr = RegInit(0.U.asTypeOf(new LqPtr)) 287 val sqPtr = RegInit(0.U.asTypeOf(new SqPtr)) 288 val lqCounter = RegInit(VirtualLoadQueueSize.U(log2Up(VirtualLoadQueueSize + 1).W)) 289 val sqCounter = RegInit(StoreQueueSize.U(log2Up(StoreQueueSize + 1).W)) 290 val canAccept = RegInit(false.B) 291 292 val loadEnqVec = io.enq.req.zip(io.enq.needAlloc).map(x => x._1.valid && x._2(0)) 293 val storeEnqVec = io.enq.req.zip(io.enq.needAlloc).map(x => x._1.valid && x._2(1)) 294 val loadEnqNumber = PopCount(loadEnqVec) 295 val storeEnqNumber = PopCount(storeEnqVec) 296 val isLastUopVec = io.enq.req.map(_.bits.lastUop) 297 val lqAllocNumber = PopCount(loadEnqVec.zip(isLastUopVec).map(x => x._1 && x._2)) 298 val sqAllocNumber = PopCount(storeEnqVec.zip(isLastUopVec).map(x => x._1 && x._2)) 299 300 // How to update ptr and counter: 301 // (1) by default, updated according to enq/commit 302 // (2) when redirect and dispatch queue is empty, update according to lsq 303 val t1_redirect = RegNext(io.redirect.valid) 304 val t2_redirect = RegNext(t1_redirect) 305 val t2_update = t2_redirect && !VecInit(io.enq.needAlloc.map(_.orR)).asUInt.orR 306 val t3_update = RegNext(t2_update) 307 val t3_lqCancelCnt = RegNext(io.lqCancelCnt) 308 val t3_sqCancelCnt = RegNext(io.sqCancelCnt) 309 when (t3_update) { 310 lqPtr := lqPtr - t3_lqCancelCnt 311 lqCounter := lqCounter + io.lcommit + t3_lqCancelCnt 312 sqPtr := sqPtr - t3_sqCancelCnt 313 sqCounter := sqCounter + io.scommit + t3_sqCancelCnt 314 }.elsewhen (!io.redirect.valid && io.enq.canAccept) { 315 lqPtr := lqPtr + lqAllocNumber 316 lqCounter := lqCounter + io.lcommit - lqAllocNumber 317 sqPtr := sqPtr + sqAllocNumber 318 sqCounter := sqCounter + io.scommit - sqAllocNumber 319 }.otherwise { 320 lqCounter := lqCounter + io.lcommit 321 sqCounter := sqCounter + io.scommit 322 } 323 324 325 val maxAllocate = backendParams.LdExuCnt max backendParams.StaExuCnt 326 val ldCanAccept = lqCounter >= lqAllocNumber +& maxAllocate.U 327 val sqCanAccept = sqCounter >= sqAllocNumber +& maxAllocate.U 328 // It is possible that t3_update and enq are true at the same clock cycle. 329 // For example, if redirect.valid lasts more than one clock cycle, 330 // after the last redirect, new instructions may enter but previously redirect 331 // has not been resolved (updated according to the cancel count from LSQ). 332 // To solve the issue easily, we block enqueue when t3_update, which is RegNext(t2_update). 333 io.enq.canAccept := RegNext(ldCanAccept && sqCanAccept && !t2_update) 334 val lqOffset = Wire(Vec(io.enq.resp.length, UInt(log2Up(maxAllocate + 1).W))) 335 val sqOffset = Wire(Vec(io.enq.resp.length, UInt(log2Up(maxAllocate + 1).W))) 336 for ((resp, i) <- io.enq.resp.zipWithIndex) { 337 lqOffset(i) := PopCount(io.enq.needAlloc.zip(isLastUopVec).take(i).map(x => x._1(0) && x._2)) 338 resp.lqIdx := lqPtr + lqOffset(i) 339 sqOffset(i) := PopCount(io.enq.needAlloc.zip(isLastUopVec).take(i).map(x => x._1(1) && x._2)) 340 resp.sqIdx := sqPtr + sqOffset(i) 341 } 342 343 io.enqLsq.needAlloc := RegNext(VecInit(io.enq.needAlloc.zip(io.enq.req).map(x => x._1 & Fill(2, x._2.bits.lastUop)))) 344 io.enqLsq.req.zip(io.enq.req).zip(io.enq.resp).foreach{ case ((toLsq, enq), resp) => 345 val do_enq = enq.valid && !io.redirect.valid && io.enq.canAccept && enq.bits.lastUop 346 toLsq.valid := RegNext(do_enq) 347 toLsq.bits := RegEnable(enq.bits, do_enq) 348 toLsq.bits.lqIdx := RegEnable(resp.lqIdx, do_enq) 349 toLsq.bits.sqIdx := RegEnable(resp.sqIdx, do_enq) 350 } 351 352}