1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chisel3._ 20import chisel3.util._ 21import difftest._ 22import difftest.common.DifftestMem 23import org.chipsalliance.cde.config.Parameters 24import utility._ 25import utils._ 26import xiangshan._ 27import xiangshan.cache._ 28import xiangshan.cache.{DCacheLineIO, DCacheWordIO, MemoryOpConstants} 29import xiangshan.backend._ 30import xiangshan.backend.rob.{RobLsqIO, RobPtr} 31import xiangshan.backend.Bundles.{DynInst, MemExuOutput} 32import xiangshan.backend.decode.isa.bitfield.{Riscv32BitInst, XSInstBitFields} 33 34class SqPtr(implicit p: Parameters) extends CircularQueuePtr[SqPtr]( 35 p => p(XSCoreParamsKey).StoreQueueSize 36){ 37} 38 39object SqPtr { 40 def apply(f: Bool, v: UInt)(implicit p: Parameters): SqPtr = { 41 val ptr = Wire(new SqPtr) 42 ptr.flag := f 43 ptr.value := v 44 ptr 45 } 46} 47 48class SqEnqIO(implicit p: Parameters) extends MemBlockBundle { 49 val canAccept = Output(Bool()) 50 val lqCanAccept = Input(Bool()) 51 val needAlloc = Vec(LSQEnqWidth, Input(Bool())) 52 val req = Vec(LSQEnqWidth, Flipped(ValidIO(new DynInst))) 53 val resp = Vec(LSQEnqWidth, Output(new SqPtr)) 54} 55 56class DataBufferEntry (implicit p: Parameters) extends DCacheBundle { 57 val addr = UInt(PAddrBits.W) 58 val vaddr = UInt(VAddrBits.W) 59 val data = UInt(VLEN.W) 60 val mask = UInt((VLEN/8).W) 61 val wline = Bool() 62 val sqPtr = new SqPtr 63 val prefetch = Bool() 64} 65 66// Store Queue 67class StoreQueue(implicit p: Parameters) extends XSModule 68 with HasDCacheParameters 69 with HasCircularQueuePtrHelper 70 with HasPerfEvents 71 with HasVLSUParameters { 72 val io = IO(new Bundle() { 73 val hartId = Input(UInt(8.W)) 74 val enq = new SqEnqIO 75 val brqRedirect = Flipped(ValidIO(new Redirect)) 76 val storeAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // store addr, data is not included 77 val vecStoreAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // vec store addr, data is not include , from vsFlowQueue 78 val storeAddrInRe = Vec(StorePipelineWidth, Input(new LsPipelineBundle())) // store more mmio and exception 79 val storeDataIn = Vec(StorePipelineWidth, Flipped(Valid(new MemExuOutput))) // store data, send to sq from rs 80 val storeMaskIn = Vec(StorePipelineWidth, Flipped(Valid(new StoreMaskBundle))) // store mask, send to sq from rs 81 val sbuffer = Vec(EnsbufferWidth, Decoupled(new DCacheWordReqWithVaddrAndPfFlag)) // write committed store to sbuffer 82 val uncacheOutstanding = Input(Bool()) 83 val mmioStout = DecoupledIO(new MemExuOutput) // writeback uncached store 84 val forward = Vec(LoadPipelineWidth, Flipped(new PipeLoadForwardQueryIO)) 85 val rob = Flipped(new RobLsqIO) 86 val uncache = new UncacheWordIO 87 // val refill = Flipped(Valid(new DCacheLineReq )) 88 val exceptionAddr = new ExceptionAddrIO 89 val sqEmpty = Output(Bool()) 90 val stAddrReadySqPtr = Output(new SqPtr) 91 val stAddrReadyVec = Output(Vec(StoreQueueSize, Bool())) 92 val stDataReadySqPtr = Output(new SqPtr) 93 val stDataReadyVec = Output(Vec(StoreQueueSize, Bool())) 94 val stIssuePtr = Output(new SqPtr) 95 val sqDeqPtr = Output(new SqPtr) 96 val sqFull = Output(Bool()) 97 val sqCancelCnt = Output(UInt(log2Up(StoreQueueSize + 1).W)) 98 val sqDeq = Output(UInt(log2Ceil(EnsbufferWidth + 1).W)) 99 val force_write = Output(Bool()) 100 val vecStoreRetire = Flipped(ValidIO(new SqPtr)) 101 }) 102 103 println("StoreQueue: size:" + StoreQueueSize) 104 105 // data modules 106 val uop = Reg(Vec(StoreQueueSize, new DynInst)) 107 // val data = Reg(Vec(StoreQueueSize, new LsqEntry)) 108 val dataModule = Module(new SQDataModule( 109 numEntries = StoreQueueSize, 110 numRead = EnsbufferWidth, 111 numWrite = StorePipelineWidth, 112 numForward = LoadPipelineWidth 113 )) 114 dataModule.io := DontCare 115 val paddrModule = Module(new SQAddrModule( 116 dataWidth = PAddrBits, 117 numEntries = StoreQueueSize, 118 numRead = EnsbufferWidth, 119 numWrite = StorePipelineWidth, 120 numForward = LoadPipelineWidth 121 )) 122 paddrModule.io := DontCare 123 val vaddrModule = Module(new SQAddrModule( 124 dataWidth = VAddrBits, 125 numEntries = StoreQueueSize, 126 numRead = EnsbufferWidth + 1, // sbuffer + badvaddr 1 (TODO) 127 numWrite = StorePipelineWidth, 128 numForward = LoadPipelineWidth 129 )) 130 vaddrModule.io := DontCare 131 val dataBuffer = Module(new DatamoduleResultBuffer(new DataBufferEntry)) 132 val debug_paddr = Reg(Vec(StoreQueueSize, UInt((PAddrBits).W))) 133 val debug_vaddr = Reg(Vec(StoreQueueSize, UInt((VAddrBits).W))) 134 val debug_data = Reg(Vec(StoreQueueSize, UInt((XLEN).W))) 135 136 // state & misc 137 val allocated = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // sq entry has been allocated 138 val addrvalid = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // non-mmio addr is valid 139 val datavalid = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // non-mmio data is valid 140 val allvalid = VecInit((0 until StoreQueueSize).map(i => addrvalid(i) && datavalid(i))) // non-mmio data & addr is valid 141 val committed = Reg(Vec(StoreQueueSize, Bool())) // inst has been committed by rob 142 val pending = Reg(Vec(StoreQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of rob 143 val mmio = Reg(Vec(StoreQueueSize, Bool())) // mmio: inst is an mmio inst 144 val atomic = Reg(Vec(StoreQueueSize, Bool())) 145 val prefetch = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // need prefetch when committing this store to sbuffer? 146 val vec = Reg(Vec(StoreQueueSize, Bool())) 147 val vecAddrvalid = Reg(Vec(StoreQueueSize, Bool())) // TODO 148 149 // ptr 150 val enqPtrExt = RegInit(VecInit((0 until io.enq.req.length).map(_.U.asTypeOf(new SqPtr)))) 151 val rdataPtrExt = RegInit(VecInit((0 until EnsbufferWidth).map(_.U.asTypeOf(new SqPtr)))) 152 val deqPtrExt = RegInit(VecInit((0 until EnsbufferWidth).map(_.U.asTypeOf(new SqPtr)))) 153 val cmtPtrExt = RegInit(VecInit((0 until CommitWidth).map(_.U.asTypeOf(new SqPtr)))) 154 val addrReadyPtrExt = RegInit(0.U.asTypeOf(new SqPtr)) 155 val dataReadyPtrExt = RegInit(0.U.asTypeOf(new SqPtr)) 156 157 val enqPtr = enqPtrExt(0).value 158 val deqPtr = deqPtrExt(0).value 159 val cmtPtr = cmtPtrExt(0).value 160 161 val validCount = distanceBetween(enqPtrExt(0), deqPtrExt(0)) 162 val allowEnqueue = validCount <= (StoreQueueSize - LSQStEnqWidth).U 163 164 val deqMask = UIntToMask(deqPtr, StoreQueueSize) 165 val enqMask = UIntToMask(enqPtr, StoreQueueSize) 166 167 val commitCount = RegNext(io.rob.scommit) 168 169 // store can be committed by ROB 170 io.rob.mmio := DontCare 171 io.rob.uop := DontCare 172 173 // Read dataModule 174 assert(EnsbufferWidth <= 2) 175 // rdataPtrExtNext and rdataPtrExtNext+1 entry will be read from dataModule 176 val rdataPtrExtNext = WireInit(Mux(dataBuffer.io.enq(1).fire, 177 VecInit(rdataPtrExt.map(_ + 2.U)), 178 Mux(dataBuffer.io.enq(0).fire || io.mmioStout.fire || io.vecStoreRetire.valid, 179 VecInit(rdataPtrExt.map(_ + 1.U)), 180 rdataPtrExt 181 ) 182 )) 183 184 // deqPtrExtNext traces which inst is about to leave store queue 185 // 186 // io.sbuffer(i).fire is RegNexted, as sbuffer data write takes 2 cycles. 187 // Before data write finish, sbuffer is unable to provide store to load 188 // forward data. As an workaround, deqPtrExt and allocated flag update 189 // is delayed so that load can get the right data from store queue. 190 // 191 // Modify deqPtrExtNext and io.sqDeq with care! 192 val deqPtrExtNext = Mux(RegNext(io.sbuffer(1).fire), 193 VecInit(deqPtrExt.map(_ + 2.U)), 194 Mux(RegNext(io.sbuffer(0).fire) || io.mmioStout.fire || io.vecStoreRetire.valid, 195 VecInit(deqPtrExt.map(_ + 1.U)), 196 deqPtrExt 197 ) 198 ) 199 io.sqDeq := RegNext(Mux(RegNext(io.sbuffer(1).fire), 2.U, 200 Mux(RegNext(io.sbuffer(0).fire) || io.mmioStout.fire || io.vecStoreRetire.valid, 1.U, 0.U) 201 )) 202 assert(!RegNext(RegNext(io.sbuffer(0).fire) && io.mmioStout.fire)) 203 204 for (i <- 0 until EnsbufferWidth) { 205 dataModule.io.raddr(i) := rdataPtrExtNext(i).value 206 paddrModule.io.raddr(i) := rdataPtrExtNext(i).value 207 vaddrModule.io.raddr(i) := rdataPtrExtNext(i).value 208 } 209 210 // no inst will be committed 1 cycle before tval update 211 vaddrModule.io.raddr(EnsbufferWidth) := (cmtPtrExt(0) + commitCount).value 212 213 val vStoreFlow = io.enq.req.map(_.bits.numLsElem) 214 val validVStoreFlow = vStoreFlow.zipWithIndex.map{case (vLoadFlowNum_Item, index) => Mux(!RegNext(io.brqRedirect.valid) && io.enq.canAccept && io.enq.lqCanAccept && canEnqueue(index), vLoadFlowNum_Item, 0.U)} 215 val validVStoreOffset = 0.U +: vStoreFlow.zip(io.enq.needAlloc) 216 .map{case (flow, needAlloc_Item) => Mux(needAlloc_Item, flow, 0.U)} 217 .slice(0, vStoreFlow.length - 1) 218 219 /** 220 * Enqueue at dispatch 221 * 222 * Currently, StoreQueue only allows enqueue when #emptyEntries > EnqWidth 223 */ 224 io.enq.canAccept := allowEnqueue 225 val canEnqueue = io.enq.req.map(_.valid) 226 val enqCancel = io.enq.req.map(_.bits.robIdx.needFlush(io.brqRedirect)) 227 for (i <- 0 until io.enq.req.length) { 228 val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i)) 229 val sqIdx = enqPtrExt(0) + validVStoreOffset.take(i + 1).reduce(_ + _) 230// val sqIdx = 0.U.asTypeOf(new SqPtr) 231 val index = io.enq.req(i).bits.sqIdx.value 232 val enqInstr = io.enq.req(i).bits.instr.asTypeOf(new XSInstBitFields) 233 when (canEnqueue(i) && !enqCancel(i)) { 234 uop(index) := io.enq.req(i).bits 235 // NOTE: the index will be used when replay 236 uop(index).sqIdx := sqIdx 237 allocated(index) := true.B 238 datavalid(index) := false.B 239 addrvalid(index) := false.B 240 committed(index) := false.B 241 pending(index) := false.B 242 prefetch(index) := false.B 243 mmio(index) := false.B 244 vec(index) := enqInstr.isVecStore // check vector store by the encoding of inst 245 vecAddrvalid(index) := false.B//TODO 246 247 XSError(!io.enq.canAccept || !io.enq.lqCanAccept, s"must accept $i\n") 248 XSError(index =/= sqIdx.value, s"must be the same entry $i\n") 249 } 250 io.enq.resp(i) := sqIdx 251 } 252 XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n") 253 254 /** 255 * Update addr/dataReadyPtr when issue from rs 256 */ 257 // update issuePtr 258 val IssuePtrMoveStride = 4 259 require(IssuePtrMoveStride >= 2) 260 261 val addrReadyLookupVec = (0 until IssuePtrMoveStride).map(addrReadyPtrExt + _.U) 262 val addrReadyLookup = addrReadyLookupVec.map(ptr => allocated(ptr.value) && (mmio(ptr.value) || addrvalid(ptr.value) || (vec(ptr.value) && vecAddrvalid(ptr.value))) && ptr =/= enqPtrExt(0)) 263 val nextAddrReadyPtr = addrReadyPtrExt + PriorityEncoder(VecInit(addrReadyLookup.map(!_) :+ true.B)) 264 addrReadyPtrExt := nextAddrReadyPtr 265 266 (0 until StoreQueueSize).map(i => { 267 io.stAddrReadyVec(i) := RegNext(allocated(i) && (mmio(i) || addrvalid(i))) 268 }) 269 270 when (io.brqRedirect.valid) { 271 addrReadyPtrExt := Mux( 272 isAfter(cmtPtrExt(0), deqPtrExt(0)), 273 cmtPtrExt(0), 274 deqPtrExtNext(0) // for mmio insts, deqPtr may be ahead of cmtPtr 275 ) 276 } 277 278 io.stAddrReadySqPtr := addrReadyPtrExt 279 280 // update 281 val dataReadyLookupVec = (0 until IssuePtrMoveStride).map(dataReadyPtrExt + _.U) 282 val dataReadyLookup = dataReadyLookupVec.map(ptr => allocated(ptr.value) && (mmio(ptr.value) || datavalid(ptr.value) || vec(ptr.value)) && ptr =/= enqPtrExt(0)) // TODO : flag of vector store data valid not add yet 283 val nextDataReadyPtr = dataReadyPtrExt + PriorityEncoder(VecInit(dataReadyLookup.map(!_) :+ true.B)) 284 dataReadyPtrExt := nextDataReadyPtr 285 286 (0 until StoreQueueSize).map(i => { 287 io.stDataReadyVec(i) := RegNext(allocated(i) && (mmio(i) || datavalid(i))) 288 }) 289 290 when (io.brqRedirect.valid) { 291 dataReadyPtrExt := Mux( 292 isAfter(cmtPtrExt(0), deqPtrExt(0)), 293 cmtPtrExt(0), 294 deqPtrExtNext(0) // for mmio insts, deqPtr may be ahead of cmtPtr 295 ) 296 } 297 298 io.stDataReadySqPtr := dataReadyPtrExt 299 io.stIssuePtr := enqPtrExt(0) 300 io.sqDeqPtr := deqPtrExt(0) 301 302 /** 303 * Writeback store from store units 304 * 305 * Most store instructions writeback to regfile in the previous cycle. 306 * However, 307 * (1) For an mmio instruction with exceptions, we need to mark it as addrvalid 308 * (in this way it will trigger an exception when it reaches ROB's head) 309 * instead of pending to avoid sending them to lower level. 310 * (2) For an mmio instruction without exceptions, we mark it as pending. 311 * When the instruction reaches ROB's head, StoreQueue sends it to uncache channel. 312 * Upon receiving the response, StoreQueue writes back the instruction 313 * through arbiter with store units. It will later commit as normal. 314 */ 315 316 // Write addr to sq 317 for (i <- 0 until StorePipelineWidth) { 318 paddrModule.io.wen(i) := false.B 319 vaddrModule.io.wen(i) := false.B 320 dataModule.io.mask.wen(i) := false.B 321 val stWbIndex = io.storeAddrIn(i).bits.uop.sqIdx.value 322 when (io.storeAddrIn(i).fire) { 323 val addr_valid = !io.storeAddrIn(i).bits.miss 324 addrvalid(stWbIndex) := addr_valid //!io.storeAddrIn(i).bits.mmio 325 // pending(stWbIndex) := io.storeAddrIn(i).bits.mmio 326 327 paddrModule.io.waddr(i) := stWbIndex 328 paddrModule.io.wdata(i) := io.storeAddrIn(i).bits.paddr 329 paddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 330 paddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 331 paddrModule.io.wen(i) := true.B 332 333 vaddrModule.io.waddr(i) := stWbIndex 334 vaddrModule.io.wdata(i) := io.storeAddrIn(i).bits.vaddr 335 vaddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 336 vaddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 337 vaddrModule.io.wen(i) := true.B 338 339 debug_paddr(paddrModule.io.waddr(i)) := paddrModule.io.wdata(i) 340 341 // mmio(stWbIndex) := io.storeAddrIn(i).bits.mmio 342 343 uop(stWbIndex) := io.storeAddrIn(i).bits.uop 344 uop(stWbIndex).debugInfo := io.storeAddrIn(i).bits.uop.debugInfo 345 XSInfo("store addr write to sq idx %d pc 0x%x miss:%d vaddr %x paddr %x mmio %x\n", 346 io.storeAddrIn(i).bits.uop.sqIdx.value, 347 io.storeAddrIn(i).bits.uop.pc, 348 io.storeAddrIn(i).bits.miss, 349 io.storeAddrIn(i).bits.vaddr, 350 io.storeAddrIn(i).bits.paddr, 351 io.storeAddrIn(i).bits.mmio 352 ) 353 } 354 355 // re-replinish mmio, for pma/pmp will get mmio one cycle later 356 val storeAddrInFireReg = RegNext(io.storeAddrIn(i).fire && !io.storeAddrIn(i).bits.miss) 357 val stWbIndexReg = RegNext(stWbIndex) 358 when (storeAddrInFireReg) { 359 pending(stWbIndexReg) := io.storeAddrInRe(i).mmio 360 mmio(stWbIndexReg) := io.storeAddrInRe(i).mmio 361 atomic(stWbIndexReg) := io.storeAddrInRe(i).atomic 362 } 363 // dcache miss info (one cycle later than storeIn) 364 // if dcache report a miss in sta pipeline, this store will trigger a prefetch when committing to sbuffer (if EnableAtCommitMissTrigger) 365 when (storeAddrInFireReg) { 366 prefetch(stWbIndexReg) := io.storeAddrInRe(i).miss 367 } 368 369 when(vaddrModule.io.wen(i)){ 370 debug_vaddr(vaddrModule.io.waddr(i)) := vaddrModule.io.wdata(i) 371 } 372 // TODO : When lastElem issue to stu or inactivative issue in vsFlowQueue, set vector store addr ready 373 val vecStWbIndex = io.vecStoreAddrIn(i).bits.uop.sqIdx.value 374 when(io.vecStoreAddrIn(i).fire){ 375 vecAddrvalid(vecStWbIndex) := true.B 376 } 377 } 378 379 // Write data to sq 380 // Now store data pipeline is actually 2 stages 381 for (i <- 0 until StorePipelineWidth) { 382 dataModule.io.data.wen(i) := false.B 383 val stWbIndex = io.storeDataIn(i).bits.uop.sqIdx.value 384 // sq data write takes 2 cycles: 385 // sq data write s0 386 when (io.storeDataIn(i).fire) { 387 // send data write req to data module 388 dataModule.io.data.waddr(i) := stWbIndex 389 dataModule.io.data.wdata(i) := Mux(io.storeDataIn(i).bits.uop.fuOpType === LSUOpType.cbo_zero, 390 0.U, 391 genWdata(io.storeDataIn(i).bits.data, io.storeDataIn(i).bits.uop.fuOpType(1,0)) 392 ) 393 dataModule.io.data.wen(i) := true.B 394 395 debug_data(dataModule.io.data.waddr(i)) := dataModule.io.data.wdata(i) 396 397 XSInfo("store data write to sq idx %d pc 0x%x data %x -> %x\n", 398 io.storeDataIn(i).bits.uop.sqIdx.value, 399 io.storeDataIn(i).bits.uop.pc, 400 io.storeDataIn(i).bits.data, 401 dataModule.io.data.wdata(i) 402 ) 403 } 404 // sq data write s1 405 when ( 406 RegNext(io.storeDataIn(i).fire) 407 // && !RegNext(io.storeDataIn(i).bits.uop).robIdx.needFlush(io.brqRedirect) 408 ) { 409 datavalid(RegNext(stWbIndex)) := true.B 410 } 411 } 412 413 // Write mask to sq 414 for (i <- 0 until StorePipelineWidth) { 415 // sq mask write s0 416 when (io.storeMaskIn(i).fire) { 417 // send data write req to data module 418 dataModule.io.mask.waddr(i) := io.storeMaskIn(i).bits.sqIdx.value 419 dataModule.io.mask.wdata(i) := io.storeMaskIn(i).bits.mask 420 dataModule.io.mask.wen(i) := true.B 421 } 422 } 423 424 /** 425 * load forward query 426 * 427 * Check store queue for instructions that is older than the load. 428 * The response will be valid at the next cycle after req. 429 */ 430 // check over all lq entries and forward data from the first matched store 431 for (i <- 0 until LoadPipelineWidth) { 432 // Compare deqPtr (deqPtr) and forward.sqIdx, we have two cases: 433 // (1) if they have the same flag, we need to check range(tail, sqIdx) 434 // (2) if they have different flags, we need to check range(tail, VirtualLoadQueueSize) and range(0, sqIdx) 435 // Forward1: Mux(same_flag, range(tail, sqIdx), range(tail, VirtualLoadQueueSize)) 436 // Forward2: Mux(same_flag, 0.U, range(0, sqIdx) ) 437 // i.e. forward1 is the target entries with the same flag bits and forward2 otherwise 438 val differentFlag = deqPtrExt(0).flag =/= io.forward(i).sqIdx.flag 439 val forwardMask = io.forward(i).sqIdxMask 440 // all addrvalid terms need to be checked 441 val addrValidVec = WireInit(VecInit((0 until StoreQueueSize).map(j => addrvalid(j) && allocated(j)))) 442 val dataValidVec = WireInit(VecInit((0 until StoreQueueSize).map(j => datavalid(j)))) 443 val allValidVec = WireInit(VecInit((0 until StoreQueueSize).map(j => addrvalid(j) && datavalid(j) && allocated(j)))) 444 445 val lfstEnable = Constantin.createRecord("LFSTEnable", LFSTEnable.B).orR 446 val storeSetHitVec = Mux(lfstEnable, 447 WireInit(VecInit((0 until StoreQueueSize).map(j => io.forward(i).uop.loadWaitBit && uop(j).robIdx === io.forward(i).uop.waitForRobIdx))), 448 WireInit(VecInit((0 until StoreQueueSize).map(j => uop(j).storeSetHit && uop(j).ssid === io.forward(i).uop.ssid))) 449 ) 450 451 val forwardMask1 = Mux(differentFlag, ~deqMask, deqMask ^ forwardMask) 452 val forwardMask2 = Mux(differentFlag, forwardMask, 0.U(StoreQueueSize.W)) 453 val canForward1 = forwardMask1 & allValidVec.asUInt 454 val canForward2 = forwardMask2 & allValidVec.asUInt 455 val needForward = Mux(differentFlag, ~deqMask | forwardMask, deqMask ^ forwardMask) 456 457 XSDebug(p"$i f1 ${Binary(canForward1)} f2 ${Binary(canForward2)} " + 458 p"sqIdx ${io.forward(i).sqIdx} pa ${Hexadecimal(io.forward(i).paddr)}\n" 459 ) 460 461 // do real fwd query (cam lookup in load_s1) 462 dataModule.io.needForward(i)(0) := canForward1 & vaddrModule.io.forwardMmask(i).asUInt 463 dataModule.io.needForward(i)(1) := canForward2 & vaddrModule.io.forwardMmask(i).asUInt 464 465 vaddrModule.io.forwardMdata(i) := io.forward(i).vaddr 466 vaddrModule.io.forwardDataMask(i) := io.forward(i).mask 467 paddrModule.io.forwardMdata(i) := io.forward(i).paddr 468 paddrModule.io.forwardDataMask(i) := io.forward(i).mask 469 470 471 // vaddr cam result does not equal to paddr cam result 472 // replay needed 473 // val vpmaskNotEqual = ((paddrModule.io.forwardMmask(i).asUInt ^ vaddrModule.io.forwardMmask(i).asUInt) & needForward) =/= 0.U 474 // val vaddrMatchFailed = vpmaskNotEqual && io.forward(i).valid 475 val vpmaskNotEqual = ( 476 (RegNext(paddrModule.io.forwardMmask(i).asUInt) ^ RegNext(vaddrModule.io.forwardMmask(i).asUInt)) & 477 RegNext(needForward) & 478 RegNext(addrValidVec.asUInt) 479 ) =/= 0.U 480 val vaddrMatchFailed = vpmaskNotEqual && RegNext(io.forward(i).valid) 481 when (vaddrMatchFailed) { 482 XSInfo("vaddrMatchFailed: pc %x pmask %x vmask %x\n", 483 RegNext(io.forward(i).uop.pc), 484 RegNext(needForward & paddrModule.io.forwardMmask(i).asUInt), 485 RegNext(needForward & vaddrModule.io.forwardMmask(i).asUInt) 486 ); 487 } 488 XSPerfAccumulate("vaddr_match_failed", vpmaskNotEqual) 489 XSPerfAccumulate("vaddr_match_really_failed", vaddrMatchFailed) 490 491 // Fast forward mask will be generated immediately (load_s1) 492 io.forward(i).forwardMaskFast := dataModule.io.forwardMaskFast(i) 493 494 // Forward result will be generated 1 cycle later (load_s2) 495 io.forward(i).forwardMask := dataModule.io.forwardMask(i) 496 io.forward(i).forwardData := dataModule.io.forwardData(i) 497 // If addr match, data not ready, mark it as dataInvalid 498 // load_s1: generate dataInvalid in load_s1 to set fastUop 499 val dataInvalidMask1 = (addrValidVec.asUInt & ~dataValidVec.asUInt & vaddrModule.io.forwardMmask(i).asUInt & forwardMask1.asUInt) 500 val dataInvalidMask2 = (addrValidVec.asUInt & ~dataValidVec.asUInt & vaddrModule.io.forwardMmask(i).asUInt & forwardMask2.asUInt) 501 val dataInvalidMask = dataInvalidMask1 | dataInvalidMask2 502 io.forward(i).dataInvalidFast := dataInvalidMask.orR 503 504 // make chisel happy 505 val dataInvalidMask1Reg = Wire(UInt(StoreQueueSize.W)) 506 dataInvalidMask1Reg := RegNext(dataInvalidMask1) 507 // make chisel happy 508 val dataInvalidMask2Reg = Wire(UInt(StoreQueueSize.W)) 509 dataInvalidMask2Reg := RegNext(dataInvalidMask2) 510 val dataInvalidMaskReg = dataInvalidMask1Reg | dataInvalidMask2Reg 511 512 // If SSID match, address not ready, mark it as addrInvalid 513 // load_s2: generate addrInvalid 514 val addrInvalidMask1 = (~addrValidVec.asUInt & storeSetHitVec.asUInt & forwardMask1.asUInt) 515 val addrInvalidMask2 = (~addrValidVec.asUInt & storeSetHitVec.asUInt & forwardMask2.asUInt) 516 // make chisel happy 517 val addrInvalidMask1Reg = Wire(UInt(StoreQueueSize.W)) 518 addrInvalidMask1Reg := RegNext(addrInvalidMask1) 519 // make chisel happy 520 val addrInvalidMask2Reg = Wire(UInt(StoreQueueSize.W)) 521 addrInvalidMask2Reg := RegNext(addrInvalidMask2) 522 val addrInvalidMaskReg = addrInvalidMask1Reg | addrInvalidMask2Reg 523 524 // load_s2 525 io.forward(i).dataInvalid := RegNext(io.forward(i).dataInvalidFast) 526 // check if vaddr forward mismatched 527 io.forward(i).matchInvalid := vaddrMatchFailed 528 529 // data invalid sq index 530 // check whether false fail 531 // check flag 532 val s2_differentFlag = RegNext(differentFlag) 533 val s2_enqPtrExt = RegNext(enqPtrExt(0)) 534 val s2_deqPtrExt = RegNext(deqPtrExt(0)) 535 536 // addr invalid sq index 537 // make chisel happy 538 val addrInvalidMaskRegWire = Wire(UInt(StoreQueueSize.W)) 539 addrInvalidMaskRegWire := addrInvalidMaskReg 540 val addrInvalidFlag = addrInvalidMaskRegWire.orR 541 val hasInvalidAddr = (~addrValidVec.asUInt & needForward).orR 542 543 val addrInvalidSqIdx1 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(addrInvalidMask1Reg)))) 544 val addrInvalidSqIdx2 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(addrInvalidMask2Reg)))) 545 val addrInvalidSqIdx = Mux(addrInvalidMask2Reg.orR, addrInvalidSqIdx2, addrInvalidSqIdx1) 546 547 // store-set content management 548 // +-----------------------+ 549 // | Search a SSID for the | 550 // | load operation | 551 // +-----------------------+ 552 // | 553 // V 554 // +-------------------+ 555 // | load wait strict? | 556 // +-------------------+ 557 // | 558 // V 559 // +----------------------+ 560 // Set| |Clean 561 // V V 562 // +------------------------+ +------------------------------+ 563 // | Waiting for all older | | Wait until the corresponding | 564 // | stores operations | | older store operations | 565 // +------------------------+ +------------------------------+ 566 567 568 569 when (RegNext(io.forward(i).uop.loadWaitStrict)) { 570 io.forward(i).addrInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx - 1.U) 571 } .elsewhen (addrInvalidFlag) { 572 io.forward(i).addrInvalidSqIdx.flag := Mux(!s2_differentFlag || addrInvalidSqIdx >= s2_deqPtrExt.value, s2_deqPtrExt.flag, s2_enqPtrExt.flag) 573 io.forward(i).addrInvalidSqIdx.value := addrInvalidSqIdx 574 } .otherwise { 575 // may be store inst has been written to sbuffer already. 576 io.forward(i).addrInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx) 577 } 578 io.forward(i).addrInvalid := Mux(RegNext(io.forward(i).uop.loadWaitStrict), RegNext(hasInvalidAddr), addrInvalidFlag) 579 580 // data invalid sq index 581 // make chisel happy 582 val dataInvalidMaskRegWire = Wire(UInt(StoreQueueSize.W)) 583 dataInvalidMaskRegWire := dataInvalidMaskReg 584 val dataInvalidFlag = dataInvalidMaskRegWire.orR 585 586 val dataInvalidSqIdx1 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(dataInvalidMask1Reg)))) 587 val dataInvalidSqIdx2 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(dataInvalidMask2Reg)))) 588 val dataInvalidSqIdx = Mux(dataInvalidMask2Reg.orR, dataInvalidSqIdx2, dataInvalidSqIdx1) 589 590 when (dataInvalidFlag) { 591 io.forward(i).dataInvalidSqIdx.flag := Mux(!s2_differentFlag || dataInvalidSqIdx >= s2_deqPtrExt.value, s2_deqPtrExt.flag, s2_enqPtrExt.flag) 592 io.forward(i).dataInvalidSqIdx.value := dataInvalidSqIdx 593 } .otherwise { 594 // may be store inst has been written to sbuffer already. 595 io.forward(i).dataInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx) 596 } 597 } 598 599 /** 600 * Memory mapped IO / other uncached operations 601 * 602 * States: 603 * (1) writeback from store units: mark as pending 604 * (2) when they reach ROB's head, they can be sent to uncache channel 605 * (3) response from uncache channel: mark as datavalidmask.wen 606 * (4) writeback to ROB (and other units): mark as writebacked 607 * (5) ROB commits the instruction: same as normal instructions 608 */ 609 //(2) when they reach ROB's head, they can be sent to uncache channel 610 val s_idle :: s_req :: s_resp :: s_wb :: s_wait :: Nil = Enum(5) 611 val uncacheState = RegInit(s_idle) 612 switch(uncacheState) { 613 is(s_idle) { 614 when(RegNext(io.rob.pendingst && pending(deqPtr) && allocated(deqPtr) && datavalid(deqPtr) && addrvalid(deqPtr))) { 615 uncacheState := s_req 616 } 617 } 618 is(s_req) { 619 when (io.uncache.req.fire) { 620 when (io.uncacheOutstanding) { 621 uncacheState := s_wb 622 } .otherwise { 623 uncacheState := s_resp 624 } 625 } 626 } 627 is(s_resp) { 628 when(io.uncache.resp.fire) { 629 uncacheState := s_wb 630 } 631 } 632 is(s_wb) { 633 when (io.mmioStout.fire) { 634 uncacheState := s_wait 635 } 636 } 637 is(s_wait) { 638 when(commitCount > 0.U) { 639 uncacheState := s_idle // ready for next mmio 640 } 641 } 642 } 643 io.uncache.req.valid := uncacheState === s_req 644 645 io.uncache.req.bits := DontCare 646 io.uncache.req.bits.cmd := MemoryOpConstants.M_XWR 647 io.uncache.req.bits.addr := paddrModule.io.rdata(0) // data(deqPtr) -> rdata(0) 648 io.uncache.req.bits.data := shiftDataToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).data) 649 io.uncache.req.bits.mask := shiftMaskToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).mask) 650 651 // CBO op type check can be delayed for 1 cycle, 652 // as uncache op will not start in s_idle 653 val cbo_mmio_addr = paddrModule.io.rdata(0) >> 2 << 2 // clear lowest 2 bits for op 654 val cbo_mmio_op = 0.U //TODO 655 val cbo_mmio_data = cbo_mmio_addr | cbo_mmio_op 656 when(RegNext(LSUOpType.isCbo(uop(deqPtr).fuOpType))){ 657 io.uncache.req.bits.addr := DontCare // TODO 658 io.uncache.req.bits.data := paddrModule.io.rdata(0) 659 io.uncache.req.bits.mask := DontCare // TODO 660 } 661 662 io.uncache.req.bits.atomic := atomic(RegNext(rdataPtrExtNext(0)).value) 663 664 when(io.uncache.req.fire){ 665 // mmio store should not be committed until uncache req is sent 666 pending(deqPtr) := false.B 667 668 XSDebug( 669 p"uncache req: pc ${Hexadecimal(uop(deqPtr).pc)} " + 670 p"addr ${Hexadecimal(io.uncache.req.bits.addr)} " + 671 p"data ${Hexadecimal(io.uncache.req.bits.data)} " + 672 p"op ${Hexadecimal(io.uncache.req.bits.cmd)} " + 673 p"mask ${Hexadecimal(io.uncache.req.bits.mask)}\n" 674 ) 675 } 676 677 // (3) response from uncache channel: mark as datavalid 678 io.uncache.resp.ready := true.B 679 680 // (4) writeback to ROB (and other units): mark as writebacked 681 io.mmioStout.valid := uncacheState === s_wb 682 io.mmioStout.bits.uop := uop(deqPtr) 683 io.mmioStout.bits.uop.sqIdx := deqPtrExt(0) 684 io.mmioStout.bits.data := shiftDataToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).data) // dataModule.io.rdata.read(deqPtr) 685 io.mmioStout.bits.debug.isMMIO := true.B 686 io.mmioStout.bits.debug.paddr := DontCare 687 io.mmioStout.bits.debug.isPerfCnt := false.B 688 io.mmioStout.bits.debug.vaddr := DontCare 689 // Remove MMIO inst from store queue after MMIO request is being sent 690 // That inst will be traced by uncache state machine 691 when (io.mmioStout.fire) { 692 allocated(deqPtr) := false.B 693 } 694 695 /** 696 * ROB commits store instructions (mark them as committed) 697 * 698 * (1) When store commits, mark it as committed. 699 * (2) They will not be cancelled and can be sent to lower level. 700 */ 701 XSError(uncacheState =/= s_idle && uncacheState =/= s_wait && commitCount > 0.U, 702 "should not commit instruction when MMIO has not been finished\n") 703 for (i <- 0 until CommitWidth) { 704 when (commitCount > i.U) { // MMIO inst is not in progress 705 if(i == 0){ 706 // MMIO inst should not update committed flag 707 // Note that commit count has been delayed for 1 cycle 708 when(uncacheState === s_idle){ 709 committed(cmtPtrExt(0).value) := true.B 710 } 711 } else { 712 committed(cmtPtrExt(i).value) := true.B 713 } 714 } 715 } 716 cmtPtrExt := cmtPtrExt.map(_ + commitCount) 717 718 // committed stores will not be cancelled and can be sent to lower level. 719 // remove retired insts from sq, add retired store to sbuffer 720 721 // Read data from data module 722 // As store queue grows larger and larger, time needed to read data from data 723 // module keeps growing higher. Now we give data read a whole cycle. 724 725 // Vector stores are written to sbuffer by vector store flow queue rather than sq 726 XSError(io.vecStoreRetire.valid && !vec(rdataPtrExt(0).value), "Vector store flow queue is trying to retire a scalar store") 727 XSError(io.vecStoreRetire.valid && !allocated(rdataPtrExt(0).value), "Vector store flow queue is trying to retire an invalid entry") 728 XSError(io.vecStoreRetire.valid && vec(rdataPtrExt(0).value) && !vecAddrvalid(rdataPtrExt(0).value), "Vector store is trying to retire without write last element!") 729 when (io.vecStoreRetire.valid) { 730 assert(io.vecStoreRetire.bits === rdataPtrExt(0)) 731 vec(rdataPtrExt(0).value) := false.B 732 vecAddrvalid(rdataPtrExt(0).value) := false.B 733 allocated(rdataPtrExt(0).value) := false.B 734 } 735 736 val mmioStall = mmio(rdataPtrExt(0).value) 737 val vecStall = vec(rdataPtrExt(0).value) 738 for (i <- 0 until EnsbufferWidth) { 739 val ptr = rdataPtrExt(i).value 740 dataBuffer.io.enq(i).valid := allocated(ptr) && committed(ptr) && !mmioStall && !vecStall 741 // Note that store data/addr should both be valid after store's commit 742 assert(!dataBuffer.io.enq(i).valid || allvalid(ptr)) 743 dataBuffer.io.enq(i).bits.addr := paddrModule.io.rdata(i) 744 dataBuffer.io.enq(i).bits.vaddr := vaddrModule.io.rdata(i) 745 dataBuffer.io.enq(i).bits.data := dataModule.io.rdata(i).data 746 dataBuffer.io.enq(i).bits.mask := dataModule.io.rdata(i).mask 747 dataBuffer.io.enq(i).bits.wline := paddrModule.io.rlineflag(i) 748 dataBuffer.io.enq(i).bits.sqPtr := rdataPtrExt(i) 749 dataBuffer.io.enq(i).bits.prefetch := prefetch(ptr) 750 } 751 752 // Send data stored in sbufferReqBitsReg to sbuffer 753 for (i <- 0 until EnsbufferWidth) { 754 io.sbuffer(i).valid := dataBuffer.io.deq(i).valid 755 dataBuffer.io.deq(i).ready := io.sbuffer(i).ready 756 // Write line request should have all 1 mask 757 assert(!(io.sbuffer(i).valid && io.sbuffer(i).bits.wline && !io.sbuffer(i).bits.mask.andR)) 758 io.sbuffer(i).bits := DontCare 759 io.sbuffer(i).bits.cmd := MemoryOpConstants.M_XWR 760 io.sbuffer(i).bits.addr := dataBuffer.io.deq(i).bits.addr 761 io.sbuffer(i).bits.vaddr := dataBuffer.io.deq(i).bits.vaddr 762 io.sbuffer(i).bits.data := dataBuffer.io.deq(i).bits.data 763 io.sbuffer(i).bits.mask := dataBuffer.io.deq(i).bits.mask 764 io.sbuffer(i).bits.wline := dataBuffer.io.deq(i).bits.wline 765 io.sbuffer(i).bits.prefetch := dataBuffer.io.deq(i).bits.prefetch 766 767 // io.sbuffer(i).fire is RegNexted, as sbuffer data write takes 2 cycles. 768 // Before data write finish, sbuffer is unable to provide store to load 769 // forward data. As an workaround, deqPtrExt and allocated flag update 770 // is delayed so that load can get the right data from store queue. 771 val ptr = dataBuffer.io.deq(i).bits.sqPtr.value 772 when (RegNext(io.sbuffer(i).fire)) { 773 allocated(RegEnable(ptr, io.sbuffer(i).fire)) := false.B 774 XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr) 775 } 776 } 777 (1 until EnsbufferWidth).foreach(i => when(io.sbuffer(i).fire) { assert(io.sbuffer(i - 1).fire) }) 778 if (coreParams.dcacheParametersOpt.isEmpty) { 779 for (i <- 0 until EnsbufferWidth) { 780 val ptr = deqPtrExt(i).value 781 val ram = DifftestMem(64L * 1024 * 1024 * 1024, 8) 782 val wen = allocated(ptr) && committed(ptr) && !mmio(ptr) 783 val waddr = ((paddrModule.io.rdata(i) - "h80000000".U) >> 3).asUInt 784 val wdata = Mux(paddrModule.io.rdata(i)(3), dataModule.io.rdata(i).data(127, 64), dataModule.io.rdata(i).data(63, 0)) 785 val wmask = Mux(paddrModule.io.rdata(i)(3), dataModule.io.rdata(i).mask(15, 8), dataModule.io.rdata(i).mask(7, 0)) 786 when (wen) { 787 ram.write(waddr, wdata.asTypeOf(Vec(8, UInt(8.W))), wmask.asBools) 788 } 789 } 790 } 791 792 // Read vaddr for mem exception 793 io.exceptionAddr.vaddr := vaddrModule.io.rdata(EnsbufferWidth) 794 795 // misprediction recovery / exception redirect 796 // invalidate sq term using robIdx 797 val needCancel = Wire(Vec(StoreQueueSize, Bool())) 798 for (i <- 0 until StoreQueueSize) { 799 needCancel(i) := uop(i).robIdx.needFlush(io.brqRedirect) && allocated(i) && !committed(i) 800 when (needCancel(i)) { 801 allocated(i) := false.B 802 } 803 } 804 805 /** 806* update pointers 807**/ 808 val lastEnqCancel = PopCount(RegNext(VecInit(canEnqueue.zip(enqCancel).map(x => x._1 && x._2)))) // 1 cycle after redirect 809 val lastCycleCancelCount = PopCount(RegNext(needCancel)) // 1 cycle after redirect 810 val lastCycleRedirect = RegNext(io.brqRedirect.valid) // 1 cycle after redirect 811 val enqNumber = validVStoreFlow.reduce(_ + _) 812 813 val lastlastCycleRedirect=RegNext(lastCycleRedirect)// 2 cycle after redirect 814 val redirectCancelCount = RegEnable(lastCycleCancelCount + lastEnqCancel, lastCycleRedirect) // 2 cycle after redirect 815 816 when (lastlastCycleRedirect) { 817 // we recover the pointers in 2 cycle after redirect for better timing 818 enqPtrExt := VecInit(enqPtrExt.map(_ - redirectCancelCount)) 819 }.otherwise { 820 // lastCycleRedirect.valid or nornal case 821 // when lastCycleRedirect.valid, enqNumber === 0.U, enqPtrExt will not change 822 enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) 823 } 824 assert(!(lastCycleRedirect && enqNumber =/= 0.U)) 825 826 deqPtrExt := deqPtrExtNext 827 rdataPtrExt := rdataPtrExtNext 828 829 // val dequeueCount = Mux(io.sbuffer(1).fire, 2.U, Mux(io.sbuffer(0).fire || io.mmioStout.fire, 1.U, 0.U)) 830 831 // If redirect at T0, sqCancelCnt is at T2 832 io.sqCancelCnt := redirectCancelCount 833 val ForceWriteUpper = Wire(UInt(log2Up(StoreQueueSize + 1).W)) 834 ForceWriteUpper := Constantin.createRecord("ForceWriteUpper_"+p(XSCoreParamsKey).HartId.toString(), initValue = 60.U) 835 val ForceWriteLower = Wire(UInt(log2Up(StoreQueueSize + 1).W)) 836 ForceWriteLower := Constantin.createRecord("ForceWriteLower_"+p(XSCoreParamsKey).HartId.toString(), initValue = 55.U) 837 838 val valid_cnt = PopCount(allocated) 839 io.force_write := RegNext(Mux(valid_cnt >= ForceWriteUpper, true.B, valid_cnt >= ForceWriteLower && io.force_write), init = false.B) 840 841 // io.sqempty will be used by sbuffer 842 // We delay it for 1 cycle for better timing 843 // When sbuffer need to check if it is empty, the pipeline is blocked, which means delay io.sqempty 844 // for 1 cycle will also promise that sq is empty in that cycle 845 io.sqEmpty := RegNext( 846 enqPtrExt(0).value === deqPtrExt(0).value && 847 enqPtrExt(0).flag === deqPtrExt(0).flag 848 ) 849 // perf counter 850 QueuePerf(StoreQueueSize, validCount, !allowEnqueue) 851 io.sqFull := !allowEnqueue 852 XSPerfAccumulate("mmioCycle", uncacheState =/= s_idle) // lq is busy dealing with uncache req 853 XSPerfAccumulate("mmioCnt", io.uncache.req.fire) 854 XSPerfAccumulate("mmio_wb_success", io.mmioStout.fire) 855 XSPerfAccumulate("mmio_wb_blocked", io.mmioStout.valid && !io.mmioStout.ready) 856 XSPerfAccumulate("validEntryCnt", distanceBetween(enqPtrExt(0), deqPtrExt(0))) 857 XSPerfAccumulate("cmtEntryCnt", distanceBetween(cmtPtrExt(0), deqPtrExt(0))) 858 XSPerfAccumulate("nCmtEntryCnt", distanceBetween(enqPtrExt(0), cmtPtrExt(0))) 859 860 val perfValidCount = distanceBetween(enqPtrExt(0), deqPtrExt(0)) 861 val perfEvents = Seq( 862 ("mmioCycle ", uncacheState =/= s_idle), 863 ("mmioCnt ", io.uncache.req.fire), 864 ("mmio_wb_success", io.mmioStout.fire), 865 ("mmio_wb_blocked", io.mmioStout.valid && !io.mmioStout.ready), 866 ("stq_1_4_valid ", (perfValidCount < (StoreQueueSize.U/4.U))), 867 ("stq_2_4_valid ", (perfValidCount > (StoreQueueSize.U/4.U)) & (perfValidCount <= (StoreQueueSize.U/2.U))), 868 ("stq_3_4_valid ", (perfValidCount > (StoreQueueSize.U/2.U)) & (perfValidCount <= (StoreQueueSize.U*3.U/4.U))), 869 ("stq_4_4_valid ", (perfValidCount > (StoreQueueSize.U*3.U/4.U))), 870 ) 871 generatePerfEvent() 872 873 // debug info 874 XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt(0).flag, deqPtr) 875 876 def PrintFlag(flag: Bool, name: String): Unit = { 877 when(flag) { 878 XSDebug(false, true.B, name) 879 }.otherwise { 880 XSDebug(false, true.B, " ") 881 } 882 } 883 884 for (i <- 0 until StoreQueueSize) { 885 XSDebug(i + ": pc %x va %x pa %x data %x ", 886 uop(i).pc, 887 debug_vaddr(i), 888 debug_paddr(i), 889 debug_data(i) 890 ) 891 PrintFlag(allocated(i), "a") 892 PrintFlag(allocated(i) && addrvalid(i), "a") 893 PrintFlag(allocated(i) && datavalid(i), "d") 894 PrintFlag(allocated(i) && committed(i), "c") 895 PrintFlag(allocated(i) && pending(i), "p") 896 PrintFlag(allocated(i) && mmio(i), "m") 897 XSDebug(false, true.B, "\n") 898 } 899 900} 901