1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan._ 25import xiangshan.cache._ 26import xiangshan.cache.{DCacheWordIO, DCacheLineIO, MemoryOpConstants} 27import xiangshan.cache.mmu.{TlbRequestIO, TlbHintIO} 28import xiangshan.mem._ 29import xiangshan.backend.rob.RobLsqIO 30 31class ExceptionAddrIO(implicit p: Parameters) extends XSBundle { 32 val isStore = Input(Bool()) 33 val vaddr = Output(UInt(VAddrBits.W)) 34 val gpaddr = Output(UInt(GPAddrBits.W)) 35} 36 37class FwdEntry extends Bundle { 38 val validFast = Bool() // validFast is generated the same cycle with query 39 val valid = Bool() // valid is generated 1 cycle after query request 40 val data = UInt(8.W) // data is generated 1 cycle after query request 41} 42 43// inflight miss block reqs 44class InflightBlockInfo(implicit p: Parameters) extends XSBundle { 45 val block_addr = UInt(PAddrBits.W) 46 val valid = Bool() 47} 48 49class LsqEnqIO(implicit p: Parameters) extends XSBundle { 50 val canAccept = Output(Bool()) 51 val needAlloc = Vec(exuParameters.LsExuCnt, Input(UInt(2.W))) 52 val req = Vec(exuParameters.LsExuCnt, Flipped(ValidIO(new MicroOp))) 53 val resp = Vec(exuParameters.LsExuCnt, Output(new LSIdx)) 54} 55 56// Load / Store Queue Wrapper for XiangShan Out of Order LSU 57class LsqWrapper(implicit p: Parameters) extends XSModule with HasDCacheParameters with HasPerfEvents { 58 val io = IO(new Bundle() { 59 val hartId = Input(UInt(hartIdLen.W)) 60 val brqRedirect = Flipped(ValidIO(new Redirect)) 61 val enq = new LsqEnqIO 62 val ldu = new Bundle() { 63 val stld_nuke_query = Vec(LoadPipelineWidth, Flipped(new LoadNukeQueryIO)) // from load_s2 64 val ldld_nuke_query = Vec(LoadPipelineWidth, Flipped(new LoadNukeQueryIO)) // from load_s2 65 val ldin = Vec(LoadPipelineWidth, Flipped(Decoupled(new LqWriteBundle))) // from load_s3 66 } 67 val sta = new Bundle() { 68 val storeMaskIn = Vec(StorePipelineWidth, Flipped(Valid(new StoreMaskBundle))) // from store_s0, store mask, send to sq from rs 69 val storeAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // from store_s1 70 val storeAddrInRe = Vec(StorePipelineWidth, Input(new LsPipelineBundle())) // from store_s2 71 } 72 val std = new Bundle() { 73 val storeDataIn = Vec(StorePipelineWidth, Flipped(Valid(new ExuOutput))) // from store_s0, store data, send to sq from rs 74 } 75 val ldout = Vec(LoadPipelineWidth, DecoupledIO(new ExuOutput)) 76 val ld_raw_data = Vec(LoadPipelineWidth, Output(new LoadDataFromLQBundle)) 77 val replay = Vec(LoadPipelineWidth, Decoupled(new LsPipelineBundle)) 78 val sbuffer = Vec(EnsbufferWidth, Decoupled(new DCacheWordReqWithVaddrAndPfFlag)) 79 val forward = Vec(LoadPipelineWidth, Flipped(new PipeLoadForwardQueryIO)) 80 val rob = Flipped(new RobLsqIO) 81 val nuke_rollback = Output(Valid(new Redirect)) 82 val nack_rollback = Output(Valid(new Redirect)) 83 val release = Flipped(Valid(new Release)) 84 // val refill = Flipped(Valid(new Refill)) 85 val tl_d_channel = Input(new DcacheToLduForwardIO) 86 val uncacheOutstanding = Input(Bool()) 87 val uncache = new UncacheWordIO 88 val mmioStout = DecoupledIO(new ExuOutput) // writeback uncached store 89 val sqEmpty = Output(Bool()) 90 val lq_rep_full = Output(Bool()) 91 val sqFull = Output(Bool()) 92 val lqFull = Output(Bool()) 93 val sqCancelCnt = Output(UInt(log2Up(StoreQueueSize+1).W)) 94 val lqCancelCnt = Output(UInt(log2Up(VirtualLoadQueueSize+1).W)) 95 val lqDeq = Output(UInt(log2Up(CommitWidth + 1).W)) 96 val sqDeq = Output(UInt(log2Ceil(EnsbufferWidth + 1).W)) 97 val lqCanAccept = Output(Bool()) 98 val sqCanAccept = Output(Bool()) 99 val exceptionAddr = new ExceptionAddrIO 100 val issuePtrExt = Output(new SqPtr) 101 val l2_hint = Input(Valid(new L2ToL1Hint())) 102 val tlb_hint = Flipped(new TlbHintIO) 103 val force_write = Output(Bool()) 104 val lqEmpty = Output(Bool()) 105 val debugTopDown = new LoadQueueTopDownIO 106 }) 107 108 val loadQueue = Module(new LoadQueue) 109 val storeQueue = Module(new StoreQueue) 110 111 storeQueue.io.hartId := io.hartId 112 storeQueue.io.uncacheOutstanding := io.uncacheOutstanding 113 114 115 dontTouch(loadQueue.io.tlbReplayDelayCycleCtrl) 116 val tlbReplayDelayCycleCtrl = WireInit(VecInit(Seq(14.U(ReSelectLen.W), 0.U(ReSelectLen.W), 125.U(ReSelectLen.W), 0.U(ReSelectLen.W)))) 117 loadQueue.io.tlbReplayDelayCycleCtrl := tlbReplayDelayCycleCtrl 118 119 // io.enq logic 120 // LSQ: send out canAccept when both load queue and store queue are ready 121 // Dispatch: send instructions to LSQ only when they are ready 122 io.enq.canAccept := loadQueue.io.enq.canAccept && storeQueue.io.enq.canAccept 123 io.lqCanAccept := loadQueue.io.enq.canAccept 124 io.sqCanAccept := storeQueue.io.enq.canAccept 125 loadQueue.io.enq.sqCanAccept := storeQueue.io.enq.canAccept 126 storeQueue.io.enq.lqCanAccept := loadQueue.io.enq.canAccept 127 for (i <- io.enq.req.indices) { 128 loadQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i)(0) 129 loadQueue.io.enq.req(i).valid := io.enq.needAlloc(i)(0) && io.enq.req(i).valid 130 loadQueue.io.enq.req(i).bits := io.enq.req(i).bits 131 loadQueue.io.enq.req(i).bits.sqIdx := storeQueue.io.enq.resp(i) 132 133 storeQueue.io.enq.needAlloc(i) := io.enq.needAlloc(i)(1) 134 storeQueue.io.enq.req(i).valid := io.enq.needAlloc(i)(1) && io.enq.req(i).valid 135 storeQueue.io.enq.req(i).bits := io.enq.req(i).bits 136 storeQueue.io.enq.req(i).bits := io.enq.req(i).bits 137 storeQueue.io.enq.req(i).bits.lqIdx := loadQueue.io.enq.resp(i) 138 139 io.enq.resp(i).lqIdx := loadQueue.io.enq.resp(i) 140 io.enq.resp(i).sqIdx := storeQueue.io.enq.resp(i) 141 } 142 143 // store queue wiring 144 storeQueue.io.brqRedirect <> io.brqRedirect 145 storeQueue.io.storeAddrIn <> io.sta.storeAddrIn // from store_s1 146 storeQueue.io.storeAddrInRe <> io.sta.storeAddrInRe // from store_s2 147 storeQueue.io.storeDataIn <> io.std.storeDataIn // from store_s0 148 storeQueue.io.storeMaskIn <> io.sta.storeMaskIn // from store_s0 149 storeQueue.io.sbuffer <> io.sbuffer 150 storeQueue.io.mmioStout <> io.mmioStout 151 storeQueue.io.rob <> io.rob 152 storeQueue.io.exceptionAddr.isStore := DontCare 153 storeQueue.io.sqCancelCnt <> io.sqCancelCnt 154 storeQueue.io.sqDeq <> io.sqDeq 155 storeQueue.io.sqEmpty <> io.sqEmpty 156 storeQueue.io.sqFull <> io.sqFull 157 storeQueue.io.forward <> io.forward // overlap forwardMask & forwardData, DO NOT CHANGE SEQUENCE 158 storeQueue.io.force_write <> io.force_write 159 160 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 161 162 // load queue wiring 163 loadQueue.io.redirect <> io.brqRedirect 164 loadQueue.io.ldu <> io.ldu 165 loadQueue.io.ldout <> io.ldout 166 loadQueue.io.ld_raw_data <> io.ld_raw_data 167 loadQueue.io.rob <> io.rob 168 loadQueue.io.nuke_rollback <> io.nuke_rollback 169 loadQueue.io.nack_rollback <> io.nack_rollback 170 loadQueue.io.replay <> io.replay 171 // loadQueue.io.refill <> io.refill 172 loadQueue.io.tl_d_channel <> io.tl_d_channel 173 loadQueue.io.release <> io.release 174 loadQueue.io.exceptionAddr.isStore := DontCare 175 loadQueue.io.lqCancelCnt <> io.lqCancelCnt 176 loadQueue.io.sq.stAddrReadySqPtr <> storeQueue.io.stAddrReadySqPtr 177 loadQueue.io.sq.stAddrReadyVec <> storeQueue.io.stAddrReadyVec 178 loadQueue.io.sq.stDataReadySqPtr <> storeQueue.io.stDataReadySqPtr 179 loadQueue.io.sq.stDataReadyVec <> storeQueue.io.stDataReadyVec 180 loadQueue.io.sq.stIssuePtr <> storeQueue.io.stIssuePtr 181 loadQueue.io.sq.sqEmpty <> storeQueue.io.sqEmpty 182 loadQueue.io.sta.storeAddrIn <> io.sta.storeAddrIn // store_s1 183 loadQueue.io.std.storeDataIn <> io.std.storeDataIn // store_s0 184 loadQueue.io.lqFull <> io.lqFull 185 loadQueue.io.lq_rep_full <> io.lq_rep_full 186 loadQueue.io.lqDeq <> io.lqDeq 187 loadQueue.io.l2_hint <> io.l2_hint 188 loadQueue.io.tlb_hint <> io.tlb_hint 189 loadQueue.io.lqEmpty <> io.lqEmpty 190 191 // rob commits for lsq is delayed for two cycles, which causes the delayed update for deqPtr in lq/sq 192 // s0: commit 193 // s1: exception find 194 // s2: exception triggered 195 // s3: ptr updated & new address 196 // address will be used at the next cycle after exception is triggered 197 io.exceptionAddr.vaddr := Mux(RegNext(io.exceptionAddr.isStore), storeQueue.io.exceptionAddr.vaddr, loadQueue.io.exceptionAddr.vaddr) 198 io.exceptionAddr.gpaddr := Mux(RegNext(io.exceptionAddr.isStore), storeQueue.io.exceptionAddr.gpaddr, loadQueue.io.exceptionAddr.gpaddr) 199 io.issuePtrExt := storeQueue.io.stAddrReadySqPtr 200 201 // naive uncache arbiter 202 val s_idle :: s_load :: s_store :: Nil = Enum(3) 203 val pendingstate = RegInit(s_idle) 204 205 switch(pendingstate){ 206 is(s_idle){ 207 when(io.uncache.req.fire){ 208 pendingstate := Mux(loadQueue.io.uncache.req.valid, s_load, 209 Mux(io.uncacheOutstanding, s_idle, s_store)) 210 } 211 } 212 is(s_load){ 213 when(io.uncache.resp.fire){ 214 pendingstate := s_idle 215 } 216 } 217 is(s_store){ 218 when(io.uncache.resp.fire){ 219 pendingstate := s_idle 220 } 221 } 222 } 223 224 loadQueue.io.uncache := DontCare 225 storeQueue.io.uncache := DontCare 226 loadQueue.io.uncache.req.ready := false.B 227 storeQueue.io.uncache.req.ready := false.B 228 loadQueue.io.uncache.resp.valid := false.B 229 storeQueue.io.uncache.resp.valid := false.B 230 when(loadQueue.io.uncache.req.valid){ 231 io.uncache.req <> loadQueue.io.uncache.req 232 }.otherwise{ 233 io.uncache.req <> storeQueue.io.uncache.req 234 } 235 when (io.uncacheOutstanding) { 236 io.uncache.resp <> loadQueue.io.uncache.resp 237 } .otherwise { 238 when(pendingstate === s_load){ 239 io.uncache.resp <> loadQueue.io.uncache.resp 240 }.otherwise{ 241 io.uncache.resp <> storeQueue.io.uncache.resp 242 } 243 } 244 245 loadQueue.io.debugTopDown <> io.debugTopDown 246 247 assert(!(loadQueue.io.uncache.req.valid && storeQueue.io.uncache.req.valid)) 248 assert(!(loadQueue.io.uncache.resp.valid && storeQueue.io.uncache.resp.valid)) 249 when (!io.uncacheOutstanding) { 250 assert(!((loadQueue.io.uncache.resp.valid || storeQueue.io.uncache.resp.valid) && pendingstate === s_idle)) 251 } 252 253 254 val perfEvents = Seq(loadQueue, storeQueue).flatMap(_.getPerfEvents) 255 generatePerfEvent() 256} 257 258class LsqEnqCtrl(implicit p: Parameters) extends XSModule { 259 val io = IO(new Bundle { 260 val redirect = Flipped(ValidIO(new Redirect)) 261 // to dispatch 262 val enq = new LsqEnqIO 263 // from `memBlock.io.lqDeq 264 val lcommit = Input(UInt(log2Up(CommitWidth + 1).W)) 265 // from `memBlock.io.sqDeq` 266 val scommit = Input(UInt(log2Ceil(EnsbufferWidth + 1).W)) 267 // from/tp lsq 268 val lqCancelCnt = Input(UInt(log2Up(VirtualLoadQueueSize + 1).W)) 269 val sqCancelCnt = Input(UInt(log2Up(StoreQueueSize + 1).W)) 270 val enqLsq = Flipped(new LsqEnqIO) 271 }) 272 273 val lqPtr = RegInit(0.U.asTypeOf(new LqPtr)) 274 val sqPtr = RegInit(0.U.asTypeOf(new SqPtr)) 275 val lqCounter = RegInit(VirtualLoadQueueSize.U(log2Up(VirtualLoadQueueSize + 1).W)) 276 val sqCounter = RegInit(StoreQueueSize.U(log2Up(StoreQueueSize + 1).W)) 277 val canAccept = RegInit(false.B) 278 279 val loadEnqNumber = PopCount(io.enq.req.zip(io.enq.needAlloc).map(x => x._1.valid && x._2(0))) 280 val storeEnqNumber = PopCount(io.enq.req.zip(io.enq.needAlloc).map(x => x._1.valid && x._2(1))) 281 282 // How to update ptr and counter: 283 // (1) by default, updated according to enq/commit 284 // (2) when redirect and dispatch queue is empty, update according to lsq 285 val t1_redirect = RegNext(io.redirect.valid) 286 val t2_redirect = RegNext(t1_redirect) 287 val t2_update = t2_redirect && !VecInit(io.enq.needAlloc.map(_.orR)).asUInt.orR 288 val t3_update = RegNext(t2_update) 289 val t3_lqCancelCnt = RegNext(io.lqCancelCnt) 290 val t3_sqCancelCnt = RegNext(io.sqCancelCnt) 291 when (t3_update) { 292 lqPtr := lqPtr - t3_lqCancelCnt 293 lqCounter := lqCounter + io.lcommit + t3_lqCancelCnt 294 sqPtr := sqPtr - t3_sqCancelCnt 295 sqCounter := sqCounter + io.scommit + t3_sqCancelCnt 296 }.elsewhen (!io.redirect.valid && io.enq.canAccept) { 297 lqPtr := lqPtr + loadEnqNumber 298 lqCounter := lqCounter + io.lcommit - loadEnqNumber 299 sqPtr := sqPtr + storeEnqNumber 300 sqCounter := sqCounter + io.scommit - storeEnqNumber 301 }.otherwise { 302 lqCounter := lqCounter + io.lcommit 303 sqCounter := sqCounter + io.scommit 304 } 305 306 307 val maxAllocate = Seq(exuParameters.LduCnt, exuParameters.StuCnt).max 308 val ldCanAccept = lqCounter >= loadEnqNumber +& maxAllocate.U 309 val sqCanAccept = sqCounter >= storeEnqNumber +& maxAllocate.U 310 // It is possible that t3_update and enq are true at the same clock cycle. 311 // For example, if redirect.valid lasts more than one clock cycle, 312 // after the last redirect, new instructions may enter but previously redirect 313 // has not been resolved (updated according to the cancel count from LSQ). 314 // To solve the issue easily, we block enqueue when t3_update, which is RegNext(t2_update). 315 io.enq.canAccept := RegNext(ldCanAccept && sqCanAccept && !t2_update) 316 val lqOffset = Wire(Vec(io.enq.resp.length, UInt(log2Up(maxAllocate + 1).W))) 317 val sqOffset = Wire(Vec(io.enq.resp.length, UInt(log2Up(maxAllocate + 1).W))) 318 for ((resp, i) <- io.enq.resp.zipWithIndex) { 319 lqOffset(i) := PopCount(io.enq.needAlloc.take(i).map(a => a(0))) 320 resp.lqIdx := lqPtr + lqOffset(i) 321 sqOffset(i) := PopCount(io.enq.needAlloc.take(i).map(a => a(1))) 322 resp.sqIdx := sqPtr + sqOffset(i) 323 } 324 325 io.enqLsq.needAlloc := RegNext(io.enq.needAlloc) 326 io.enqLsq.req.zip(io.enq.req).zip(io.enq.resp).foreach{ case ((toLsq, enq), resp) => 327 val do_enq = enq.valid && !io.redirect.valid && io.enq.canAccept 328 toLsq.valid := RegNext(do_enq) 329 toLsq.bits := RegEnable(enq.bits, do_enq) 330 toLsq.bits.lqIdx := RegEnable(resp.lqIdx, do_enq) 331 toLsq.bits.sqIdx := RegEnable(resp.sqIdx, do_enq) 332 } 333 334}