1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chisel3._ 20import chisel3.util._ 21import difftest._ 22import difftest.common.DifftestMem 23import org.chipsalliance.cde.config.Parameters 24import utility._ 25import utils._ 26import xiangshan._ 27import xiangshan.cache._ 28import xiangshan.cache.{DCacheLineIO, DCacheWordIO, MemoryOpConstants} 29import xiangshan.backend._ 30import xiangshan.backend.rob.{RobLsqIO, RobPtr} 31import xiangshan.backend.Bundles.{DynInst, MemExuOutput} 32import xiangshan.backend.decode.isa.bitfield.{Riscv32BitInst, XSInstBitFields} 33 34class SqPtr(implicit p: Parameters) extends CircularQueuePtr[SqPtr]( 35 p => p(XSCoreParamsKey).StoreQueueSize 36){ 37} 38 39object SqPtr { 40 def apply(f: Bool, v: UInt)(implicit p: Parameters): SqPtr = { 41 val ptr = Wire(new SqPtr) 42 ptr.flag := f 43 ptr.value := v 44 ptr 45 } 46} 47 48class SqEnqIO(implicit p: Parameters) extends MemBlockBundle { 49 val canAccept = Output(Bool()) 50 val lqCanAccept = Input(Bool()) 51 val needAlloc = Vec(LSQEnqWidth, Input(Bool())) 52 val req = Vec(LSQEnqWidth, Flipped(ValidIO(new DynInst))) 53 val resp = Vec(LSQEnqWidth, Output(new SqPtr)) 54} 55 56class DataBufferEntry (implicit p: Parameters) extends DCacheBundle { 57 val addr = UInt(PAddrBits.W) 58 val vaddr = UInt(VAddrBits.W) 59 val data = UInt(VLEN.W) 60 val mask = UInt((VLEN/8).W) 61 val wline = Bool() 62 val sqPtr = new SqPtr 63 val prefetch = Bool() 64} 65 66// Store Queue 67class StoreQueue(implicit p: Parameters) extends XSModule 68 with HasDCacheParameters with HasCircularQueuePtrHelper with HasPerfEvents { 69 val io = IO(new Bundle() { 70 val hartId = Input(UInt(hartIdLen.W)) 71 val enq = new SqEnqIO 72 val brqRedirect = Flipped(ValidIO(new Redirect)) 73 val storeAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // store addr, data is not included 74 val vecStoreAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // vec store addr, data is not include , from vsFlowQueue 75 val storeAddrInRe = Vec(StorePipelineWidth, Input(new LsPipelineBundle())) // store more mmio and exception 76 val storeDataIn = Vec(StorePipelineWidth, Flipped(Valid(new MemExuOutput))) // store data, send to sq from rs 77 val storeMaskIn = Vec(StorePipelineWidth, Flipped(Valid(new StoreMaskBundle))) // store mask, send to sq from rs 78 val sbuffer = Vec(EnsbufferWidth, Decoupled(new DCacheWordReqWithVaddrAndPfFlag)) // write committed store to sbuffer 79 val uncacheOutstanding = Input(Bool()) 80 val mmioStout = DecoupledIO(new MemExuOutput) // writeback uncached store 81 val forward = Vec(LoadPipelineWidth, Flipped(new PipeLoadForwardQueryIO)) 82 val rob = Flipped(new RobLsqIO) 83 val uncache = new UncacheWordIO 84 // val refill = Flipped(Valid(new DCacheLineReq )) 85 val exceptionAddr = new ExceptionAddrIO 86 val sqEmpty = Output(Bool()) 87 val stAddrReadySqPtr = Output(new SqPtr) 88 val stAddrReadyVec = Output(Vec(StoreQueueSize, Bool())) 89 val stDataReadySqPtr = Output(new SqPtr) 90 val stDataReadyVec = Output(Vec(StoreQueueSize, Bool())) 91 val stIssuePtr = Output(new SqPtr) 92 val sqDeqPtr = Output(new SqPtr) 93 val sqFull = Output(Bool()) 94 val sqCancelCnt = Output(UInt(log2Up(StoreQueueSize + 1).W)) 95 val sqDeq = Output(UInt(log2Ceil(EnsbufferWidth + 1).W)) 96 val force_write = Output(Bool()) 97 val vecStoreRetire = Flipped(ValidIO(new SqPtr)) 98 }) 99 100 println("StoreQueue: size:" + StoreQueueSize) 101 102 // data modules 103 val uop = Reg(Vec(StoreQueueSize, new DynInst)) 104 // val data = Reg(Vec(StoreQueueSize, new LsqEntry)) 105 val dataModule = Module(new SQDataModule( 106 numEntries = StoreQueueSize, 107 numRead = EnsbufferWidth, 108 numWrite = StorePipelineWidth, 109 numForward = LoadPipelineWidth 110 )) 111 dataModule.io := DontCare 112 val paddrModule = Module(new SQAddrModule( 113 dataWidth = PAddrBits, 114 numEntries = StoreQueueSize, 115 numRead = EnsbufferWidth, 116 numWrite = StorePipelineWidth, 117 numForward = LoadPipelineWidth 118 )) 119 paddrModule.io := DontCare 120 val vaddrModule = Module(new SQAddrModule( 121 dataWidth = VAddrBits, 122 numEntries = StoreQueueSize, 123 numRead = EnsbufferWidth + 1, // sbuffer + badvaddr 1 (TODO) 124 numWrite = StorePipelineWidth, 125 numForward = LoadPipelineWidth 126 )) 127 val gpaddrModule = Module(new SQAddrModule( 128 dataWidth = GPAddrBits, 129 numEntries = StoreQueueSize, 130 numRead = EnsbufferWidth + 1, 131 numWrite = StorePipelineWidth, 132 numForward = LoadPipelineWidth 133 )) 134 vaddrModule.io := DontCare 135 gpaddrModule.io := DontCare 136 val dataBuffer = Module(new DatamoduleResultBuffer(new DataBufferEntry)) 137 val debug_paddr = Reg(Vec(StoreQueueSize, UInt((PAddrBits).W))) 138 val debug_vaddr = Reg(Vec(StoreQueueSize, UInt((VAddrBits).W))) 139 val debug_data = Reg(Vec(StoreQueueSize, UInt((XLEN).W))) 140 141 // state & misc 142 val allocated = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // sq entry has been allocated 143 val addrvalid = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // non-mmio addr is valid 144 val datavalid = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // non-mmio data is valid 145 val allvalid = VecInit((0 until StoreQueueSize).map(i => addrvalid(i) && datavalid(i))) // non-mmio data & addr is valid 146 val committed = Reg(Vec(StoreQueueSize, Bool())) // inst has been committed by rob 147 val pending = Reg(Vec(StoreQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of rob 148 val mmio = Reg(Vec(StoreQueueSize, Bool())) // mmio: inst is an mmio inst 149 val atomic = Reg(Vec(StoreQueueSize, Bool())) 150 val prefetch = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // need prefetch when committing this store to sbuffer? 151 val vec = Reg(Vec(StoreQueueSize, Bool())) 152 val vecAddrvalid = Reg(Vec(StoreQueueSize, Bool())) // TODO 153 154 // ptr 155 val enqPtrExt = RegInit(VecInit((0 until io.enq.req.length).map(_.U.asTypeOf(new SqPtr)))) 156 val rdataPtrExt = RegInit(VecInit((0 until EnsbufferWidth).map(_.U.asTypeOf(new SqPtr)))) 157 val deqPtrExt = RegInit(VecInit((0 until EnsbufferWidth).map(_.U.asTypeOf(new SqPtr)))) 158 val cmtPtrExt = RegInit(VecInit((0 until CommitWidth).map(_.U.asTypeOf(new SqPtr)))) 159 val addrReadyPtrExt = RegInit(0.U.asTypeOf(new SqPtr)) 160 val dataReadyPtrExt = RegInit(0.U.asTypeOf(new SqPtr)) 161 val validCounter = RegInit(0.U(log2Ceil(VirtualLoadQueueSize + 1).W)) 162 163 val enqPtr = enqPtrExt(0).value 164 val deqPtr = deqPtrExt(0).value 165 val cmtPtr = cmtPtrExt(0).value 166 167 val validCount = distanceBetween(enqPtrExt(0), deqPtrExt(0)) 168 val allowEnqueue = validCount <= (StoreQueueSize - LSQStEnqWidth).U 169 170 val deqMask = UIntToMask(deqPtr, StoreQueueSize) 171 val enqMask = UIntToMask(enqPtr, StoreQueueSize) 172 173 val commitCount = RegNext(io.rob.scommit) 174 175 // store can be committed by ROB 176 io.rob.mmio := DontCare 177 io.rob.uop := DontCare 178 179 // Read dataModule 180 assert(EnsbufferWidth <= 2) 181 // rdataPtrExtNext and rdataPtrExtNext+1 entry will be read from dataModule 182 val rdataPtrExtNext = WireInit(Mux(dataBuffer.io.enq(1).fire, 183 VecInit(rdataPtrExt.map(_ + 2.U)), 184 Mux(dataBuffer.io.enq(0).fire || io.mmioStout.fire || io.vecStoreRetire.valid, 185 VecInit(rdataPtrExt.map(_ + 1.U)), 186 rdataPtrExt 187 ) 188 )) 189 190 // deqPtrExtNext traces which inst is about to leave store queue 191 // 192 // io.sbuffer(i).fire is RegNexted, as sbuffer data write takes 2 cycles. 193 // Before data write finish, sbuffer is unable to provide store to load 194 // forward data. As an workaround, deqPtrExt and allocated flag update 195 // is delayed so that load can get the right data from store queue. 196 // 197 // Modify deqPtrExtNext and io.sqDeq with care! 198 val deqPtrExtNext = Mux(RegNext(io.sbuffer(1).fire), 199 VecInit(deqPtrExt.map(_ + 2.U)), 200 Mux(RegNext(io.sbuffer(0).fire) || io.mmioStout.fire || io.vecStoreRetire.valid, 201 VecInit(deqPtrExt.map(_ + 1.U)), 202 deqPtrExt 203 ) 204 ) 205 io.sqDeq := RegNext(Mux(RegNext(io.sbuffer(1).fire), 2.U, 206 Mux(RegNext(io.sbuffer(0).fire) || io.mmioStout.fire || io.vecStoreRetire.valid, 1.U, 0.U) 207 )) 208 assert(!RegNext(RegNext(io.sbuffer(0).fire) && io.mmioStout.fire)) 209 210 for (i <- 0 until EnsbufferWidth) { 211 dataModule.io.raddr(i) := rdataPtrExtNext(i).value 212 paddrModule.io.raddr(i) := rdataPtrExtNext(i).value 213 vaddrModule.io.raddr(i) := rdataPtrExtNext(i).value 214 gpaddrModule.io.raddr(i) := rdataPtrExtNext(i).value 215 } 216 217 // no inst will be committed 1 cycle before tval update 218 vaddrModule.io.raddr(EnsbufferWidth) := (cmtPtrExt(0) + commitCount).value 219 gpaddrModule.io.raddr(EnsbufferWidth) := (cmtPtrExt(0) + commitCount).value 220 /** 221 * Enqueue at dispatch 222 * 223 * Currently, StoreQueue only allows enqueue when #emptyEntries > EnqWidth 224 */ 225 io.enq.canAccept := allowEnqueue 226 val canEnqueue = io.enq.req.map(_.valid) 227 val enqCancel = io.enq.req.map(_.bits.robIdx.needFlush(io.brqRedirect)) 228 for (i <- 0 until io.enq.req.length) { 229 val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i)) 230 val sqIdx = enqPtrExt(offset) 231 val index = io.enq.req(i).bits.sqIdx.value 232 val enqInstr = io.enq.req(i).bits.instr.asTypeOf(new XSInstBitFields) 233 when (canEnqueue(i) && !enqCancel(i)) { 234 uop(index) := io.enq.req(i).bits 235 // NOTE: the index will be used when replay 236 uop(index).sqIdx := sqIdx 237 allocated(index) := true.B 238 datavalid(index) := false.B 239 addrvalid(index) := false.B 240 committed(index) := false.B 241 pending(index) := false.B 242 prefetch(index) := false.B 243 mmio(index) := false.B 244 vec(index) := enqInstr.isVecStore // check vector store by the encoding of inst 245 vecAddrvalid(index) := false.B//TODO 246 247 XSError(!io.enq.canAccept || !io.enq.lqCanAccept, s"must accept $i\n") 248 XSError(index =/= sqIdx.value, s"must be the same entry $i\n") 249 } 250 io.enq.resp(i) := sqIdx 251 } 252 XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n") 253 254 /** 255 * Update addr/dataReadyPtr when issue from rs 256 */ 257 // update issuePtr 258 val IssuePtrMoveStride = 4 259 require(IssuePtrMoveStride >= 2) 260 261 val addrReadyLookupVec = (0 until IssuePtrMoveStride).map(addrReadyPtrExt + _.U) 262 val addrReadyLookup = addrReadyLookupVec.map(ptr => allocated(ptr.value) && (mmio(ptr.value) || addrvalid(ptr.value) || (vec(ptr.value) && vecAddrvalid(ptr.value))) && ptr =/= enqPtrExt(0)) 263 val nextAddrReadyPtr = addrReadyPtrExt + PriorityEncoder(VecInit(addrReadyLookup.map(!_) :+ true.B)) 264 addrReadyPtrExt := nextAddrReadyPtr 265 266 (0 until StoreQueueSize).map(i => { 267 io.stAddrReadyVec(i) := RegNext(allocated(i) && (mmio(i) || addrvalid(i))) 268 }) 269 270 when (io.brqRedirect.valid) { 271 addrReadyPtrExt := Mux( 272 isAfter(cmtPtrExt(0), deqPtrExt(0)), 273 cmtPtrExt(0), 274 deqPtrExtNext(0) // for mmio insts, deqPtr may be ahead of cmtPtr 275 ) 276 } 277 278 io.stAddrReadySqPtr := addrReadyPtrExt 279 280 // update 281 val dataReadyLookupVec = (0 until IssuePtrMoveStride).map(dataReadyPtrExt + _.U) 282 val dataReadyLookup = dataReadyLookupVec.map(ptr => allocated(ptr.value) && (mmio(ptr.value) || datavalid(ptr.value) || vec(ptr.value)) && ptr =/= enqPtrExt(0)) // TODO : flag of vector store data valid not add yet 283 val nextDataReadyPtr = dataReadyPtrExt + PriorityEncoder(VecInit(dataReadyLookup.map(!_) :+ true.B)) 284 dataReadyPtrExt := nextDataReadyPtr 285 286 (0 until StoreQueueSize).map(i => { 287 io.stDataReadyVec(i) := RegNext(allocated(i) && (mmio(i) || datavalid(i))) 288 }) 289 290 when (io.brqRedirect.valid) { 291 dataReadyPtrExt := Mux( 292 isAfter(cmtPtrExt(0), deqPtrExt(0)), 293 cmtPtrExt(0), 294 deqPtrExtNext(0) // for mmio insts, deqPtr may be ahead of cmtPtr 295 ) 296 } 297 298 io.stDataReadySqPtr := dataReadyPtrExt 299 io.stIssuePtr := enqPtrExt(0) 300 io.sqDeqPtr := deqPtrExt(0) 301 302 /** 303 * Writeback store from store units 304 * 305 * Most store instructions writeback to regfile in the previous cycle. 306 * However, 307 * (1) For an mmio instruction with exceptions, we need to mark it as addrvalid 308 * (in this way it will trigger an exception when it reaches ROB's head) 309 * instead of pending to avoid sending them to lower level. 310 * (2) For an mmio instruction without exceptions, we mark it as pending. 311 * When the instruction reaches ROB's head, StoreQueue sends it to uncache channel. 312 * Upon receiving the response, StoreQueue writes back the instruction 313 * through arbiter with store units. It will later commit as normal. 314 */ 315 316 // Write addr to sq 317 for (i <- 0 until StorePipelineWidth) { 318 paddrModule.io.wen(i) := false.B 319 vaddrModule.io.wen(i) := false.B 320 gpaddrModule.io.wen(i) := false.B 321 dataModule.io.mask.wen(i) := false.B 322 val stWbIndex = io.storeAddrIn(i).bits.uop.sqIdx.value 323 when (io.storeAddrIn(i).fire) { 324 val addr_valid = !io.storeAddrIn(i).bits.miss 325 addrvalid(stWbIndex) := addr_valid //!io.storeAddrIn(i).bits.mmio 326 // pending(stWbIndex) := io.storeAddrIn(i).bits.mmio 327 328 paddrModule.io.waddr(i) := stWbIndex 329 paddrModule.io.wdata(i) := io.storeAddrIn(i).bits.paddr 330 paddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 331 paddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 332 paddrModule.io.wen(i) := true.B 333 334 vaddrModule.io.waddr(i) := stWbIndex 335 vaddrModule.io.wdata(i) := io.storeAddrIn(i).bits.vaddr 336 vaddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 337 vaddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 338 vaddrModule.io.wen(i) := true.B 339 340 gpaddrModule.io.waddr(i) := stWbIndex 341 gpaddrModule.io.wdata(i) := io.storeAddrIn(i).bits.gpaddr 342 gpaddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 343 gpaddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 344 gpaddrModule.io.wen(i) := true.B 345 346 debug_paddr(paddrModule.io.waddr(i)) := paddrModule.io.wdata(i) 347 348 // mmio(stWbIndex) := io.storeAddrIn(i).bits.mmio 349 350 uop(stWbIndex) := io.storeAddrIn(i).bits.uop 351 uop(stWbIndex).debugInfo := io.storeAddrIn(i).bits.uop.debugInfo 352 XSInfo("store addr write to sq idx %d pc 0x%x miss:%d vaddr %x paddr %x mmio %x\n", 353 io.storeAddrIn(i).bits.uop.sqIdx.value, 354 io.storeAddrIn(i).bits.uop.pc, 355 io.storeAddrIn(i).bits.miss, 356 io.storeAddrIn(i).bits.vaddr, 357 io.storeAddrIn(i).bits.paddr, 358 io.storeAddrIn(i).bits.mmio 359 ) 360 } 361 362 // re-replinish mmio, for pma/pmp will get mmio one cycle later 363 val storeAddrInFireReg = RegNext(io.storeAddrIn(i).fire && !io.storeAddrIn(i).bits.miss) 364 val stWbIndexReg = RegNext(stWbIndex) 365 when (storeAddrInFireReg) { 366 pending(stWbIndexReg) := io.storeAddrInRe(i).mmio 367 mmio(stWbIndexReg) := io.storeAddrInRe(i).mmio 368 atomic(stWbIndexReg) := io.storeAddrInRe(i).atomic 369 } 370 // dcache miss info (one cycle later than storeIn) 371 // if dcache report a miss in sta pipeline, this store will trigger a prefetch when committing to sbuffer (if EnableAtCommitMissTrigger) 372 when (storeAddrInFireReg) { 373 prefetch(stWbIndexReg) := io.storeAddrInRe(i).miss 374 } 375 376 when(vaddrModule.io.wen(i)){ 377 debug_vaddr(vaddrModule.io.waddr(i)) := vaddrModule.io.wdata(i) 378 } 379 // TODO : When lastElem issue to stu or inactivative issue in vsFlowQueue, set vector store addr ready 380 val vecStWbIndex = io.vecStoreAddrIn(i).bits.uop.sqIdx.value 381 when(io.vecStoreAddrIn(i).fire){ 382 vecAddrvalid(vecStWbIndex) := true.B 383 } 384 } 385 386 // Write data to sq 387 // Now store data pipeline is actually 2 stages 388 for (i <- 0 until StorePipelineWidth) { 389 dataModule.io.data.wen(i) := false.B 390 val stWbIndex = io.storeDataIn(i).bits.uop.sqIdx.value 391 // sq data write takes 2 cycles: 392 // sq data write s0 393 when (io.storeDataIn(i).fire) { 394 // send data write req to data module 395 dataModule.io.data.waddr(i) := stWbIndex 396 dataModule.io.data.wdata(i) := Mux(io.storeDataIn(i).bits.uop.fuOpType === LSUOpType.cbo_zero, 397 0.U, 398 genWdata(io.storeDataIn(i).bits.data, io.storeDataIn(i).bits.uop.fuOpType(1,0)) 399 ) 400 dataModule.io.data.wen(i) := true.B 401 402 debug_data(dataModule.io.data.waddr(i)) := dataModule.io.data.wdata(i) 403 404 XSInfo("store data write to sq idx %d pc 0x%x data %x -> %x\n", 405 io.storeDataIn(i).bits.uop.sqIdx.value, 406 io.storeDataIn(i).bits.uop.pc, 407 io.storeDataIn(i).bits.data, 408 dataModule.io.data.wdata(i) 409 ) 410 } 411 // sq data write s1 412 when ( 413 RegNext(io.storeDataIn(i).fire) 414 // && !RegNext(io.storeDataIn(i).bits.uop).robIdx.needFlush(io.brqRedirect) 415 ) { 416 datavalid(RegNext(stWbIndex)) := true.B 417 } 418 } 419 420 // Write mask to sq 421 for (i <- 0 until StorePipelineWidth) { 422 // sq mask write s0 423 when (io.storeMaskIn(i).fire) { 424 // send data write req to data module 425 dataModule.io.mask.waddr(i) := io.storeMaskIn(i).bits.sqIdx.value 426 dataModule.io.mask.wdata(i) := io.storeMaskIn(i).bits.mask 427 dataModule.io.mask.wen(i) := true.B 428 } 429 } 430 431 /** 432 * load forward query 433 * 434 * Check store queue for instructions that is older than the load. 435 * The response will be valid at the next cycle after req. 436 */ 437 // check over all lq entries and forward data from the first matched store 438 for (i <- 0 until LoadPipelineWidth) { 439 // Compare deqPtr (deqPtr) and forward.sqIdx, we have two cases: 440 // (1) if they have the same flag, we need to check range(tail, sqIdx) 441 // (2) if they have different flags, we need to check range(tail, VirtualLoadQueueSize) and range(0, sqIdx) 442 // Forward1: Mux(same_flag, range(tail, sqIdx), range(tail, VirtualLoadQueueSize)) 443 // Forward2: Mux(same_flag, 0.U, range(0, sqIdx) ) 444 // i.e. forward1 is the target entries with the same flag bits and forward2 otherwise 445 val differentFlag = deqPtrExt(0).flag =/= io.forward(i).sqIdx.flag 446 val forwardMask = io.forward(i).sqIdxMask 447 // all addrvalid terms need to be checked 448 val addrValidVec = WireInit(VecInit((0 until StoreQueueSize).map(j => addrvalid(j) && allocated(j)))) 449 val dataValidVec = WireInit(VecInit((0 until StoreQueueSize).map(j => datavalid(j)))) 450 val allValidVec = WireInit(VecInit((0 until StoreQueueSize).map(j => addrvalid(j) && datavalid(j) && allocated(j)))) 451 452 val lfstEnable = Constantin.createRecord("LFSTEnable", LFSTEnable) 453 val storeSetHitVec = Mux(lfstEnable, 454 WireInit(VecInit((0 until StoreQueueSize).map(j => io.forward(i).uop.loadWaitBit && uop(j).robIdx === io.forward(i).uop.waitForRobIdx))), 455 WireInit(VecInit((0 until StoreQueueSize).map(j => uop(j).storeSetHit && uop(j).ssid === io.forward(i).uop.ssid))) 456 ) 457 458 val forwardMask1 = Mux(differentFlag, ~deqMask, deqMask ^ forwardMask) 459 val forwardMask2 = Mux(differentFlag, forwardMask, 0.U(StoreQueueSize.W)) 460 val canForward1 = forwardMask1 & allValidVec.asUInt 461 val canForward2 = forwardMask2 & allValidVec.asUInt 462 val needForward = Mux(differentFlag, ~deqMask | forwardMask, deqMask ^ forwardMask) 463 464 XSDebug(p"$i f1 ${Binary(canForward1)} f2 ${Binary(canForward2)} " + 465 p"sqIdx ${io.forward(i).sqIdx} pa ${Hexadecimal(io.forward(i).paddr)}\n" 466 ) 467 468 // do real fwd query (cam lookup in load_s1) 469 dataModule.io.needForward(i)(0) := canForward1 & vaddrModule.io.forwardMmask(i).asUInt 470 dataModule.io.needForward(i)(1) := canForward2 & vaddrModule.io.forwardMmask(i).asUInt 471 472 vaddrModule.io.forwardMdata(i) := io.forward(i).vaddr 473 vaddrModule.io.forwardDataMask(i) := io.forward(i).mask 474 paddrModule.io.forwardMdata(i) := io.forward(i).paddr 475 paddrModule.io.forwardDataMask(i) := io.forward(i).mask 476 477 478 // vaddr cam result does not equal to paddr cam result 479 // replay needed 480 // val vpmaskNotEqual = ((paddrModule.io.forwardMmask(i).asUInt ^ vaddrModule.io.forwardMmask(i).asUInt) & needForward) =/= 0.U 481 // val vaddrMatchFailed = vpmaskNotEqual && io.forward(i).valid 482 val vpmaskNotEqual = ( 483 (RegNext(paddrModule.io.forwardMmask(i).asUInt) ^ RegNext(vaddrModule.io.forwardMmask(i).asUInt)) & 484 RegNext(needForward) & 485 RegNext(addrValidVec.asUInt) 486 ) =/= 0.U 487 val vaddrMatchFailed = vpmaskNotEqual && RegNext(io.forward(i).valid) 488 when (vaddrMatchFailed) { 489 XSInfo("vaddrMatchFailed: pc %x pmask %x vmask %x\n", 490 RegNext(io.forward(i).uop.pc), 491 RegNext(needForward & paddrModule.io.forwardMmask(i).asUInt), 492 RegNext(needForward & vaddrModule.io.forwardMmask(i).asUInt) 493 ); 494 } 495 XSPerfAccumulate("vaddr_match_failed", vpmaskNotEqual) 496 XSPerfAccumulate("vaddr_match_really_failed", vaddrMatchFailed) 497 498 // Fast forward mask will be generated immediately (load_s1) 499 io.forward(i).forwardMaskFast := dataModule.io.forwardMaskFast(i) 500 501 // Forward result will be generated 1 cycle later (load_s2) 502 io.forward(i).forwardMask := dataModule.io.forwardMask(i) 503 io.forward(i).forwardData := dataModule.io.forwardData(i) 504 // If addr match, data not ready, mark it as dataInvalid 505 // load_s1: generate dataInvalid in load_s1 to set fastUop 506 val dataInvalidMask1 = (addrValidVec.asUInt & ~dataValidVec.asUInt & vaddrModule.io.forwardMmask(i).asUInt & forwardMask1.asUInt) 507 val dataInvalidMask2 = (addrValidVec.asUInt & ~dataValidVec.asUInt & vaddrModule.io.forwardMmask(i).asUInt & forwardMask2.asUInt) 508 val dataInvalidMask = dataInvalidMask1 | dataInvalidMask2 509 io.forward(i).dataInvalidFast := dataInvalidMask.orR 510 511 // make chisel happy 512 val dataInvalidMask1Reg = Wire(UInt(StoreQueueSize.W)) 513 dataInvalidMask1Reg := RegNext(dataInvalidMask1) 514 // make chisel happy 515 val dataInvalidMask2Reg = Wire(UInt(StoreQueueSize.W)) 516 dataInvalidMask2Reg := RegNext(dataInvalidMask2) 517 val dataInvalidMaskReg = dataInvalidMask1Reg | dataInvalidMask2Reg 518 519 // If SSID match, address not ready, mark it as addrInvalid 520 // load_s2: generate addrInvalid 521 val addrInvalidMask1 = (~addrValidVec.asUInt & storeSetHitVec.asUInt & forwardMask1.asUInt) 522 val addrInvalidMask2 = (~addrValidVec.asUInt & storeSetHitVec.asUInt & forwardMask2.asUInt) 523 // make chisel happy 524 val addrInvalidMask1Reg = Wire(UInt(StoreQueueSize.W)) 525 addrInvalidMask1Reg := RegNext(addrInvalidMask1) 526 // make chisel happy 527 val addrInvalidMask2Reg = Wire(UInt(StoreQueueSize.W)) 528 addrInvalidMask2Reg := RegNext(addrInvalidMask2) 529 val addrInvalidMaskReg = addrInvalidMask1Reg | addrInvalidMask2Reg 530 531 // load_s2 532 io.forward(i).dataInvalid := RegNext(io.forward(i).dataInvalidFast) 533 // check if vaddr forward mismatched 534 io.forward(i).matchInvalid := vaddrMatchFailed 535 536 // data invalid sq index 537 // check whether false fail 538 // check flag 539 val s2_differentFlag = RegNext(differentFlag) 540 val s2_enqPtrExt = RegNext(enqPtrExt(0)) 541 val s2_deqPtrExt = RegNext(deqPtrExt(0)) 542 543 // addr invalid sq index 544 // make chisel happy 545 val addrInvalidMaskRegWire = Wire(UInt(StoreQueueSize.W)) 546 addrInvalidMaskRegWire := addrInvalidMaskReg 547 val addrInvalidFlag = addrInvalidMaskRegWire.orR 548 val hasInvalidAddr = (~addrValidVec.asUInt & needForward).orR 549 550 val addrInvalidSqIdx1 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(addrInvalidMask1Reg)))) 551 val addrInvalidSqIdx2 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(addrInvalidMask2Reg)))) 552 val addrInvalidSqIdx = Mux(addrInvalidMask2Reg.orR, addrInvalidSqIdx2, addrInvalidSqIdx1) 553 554 // store-set content management 555 // +-----------------------+ 556 // | Search a SSID for the | 557 // | load operation | 558 // +-----------------------+ 559 // | 560 // V 561 // +-------------------+ 562 // | load wait strict? | 563 // +-------------------+ 564 // | 565 // V 566 // +----------------------+ 567 // Set| |Clean 568 // V V 569 // +------------------------+ +------------------------------+ 570 // | Waiting for all older | | Wait until the corresponding | 571 // | stores operations | | older store operations | 572 // +------------------------+ +------------------------------+ 573 574 575 576 when (RegNext(io.forward(i).uop.loadWaitStrict)) { 577 io.forward(i).addrInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx - 1.U) 578 } .elsewhen (addrInvalidFlag) { 579 io.forward(i).addrInvalidSqIdx.flag := Mux(!s2_differentFlag || addrInvalidSqIdx >= s2_deqPtrExt.value, s2_deqPtrExt.flag, s2_enqPtrExt.flag) 580 io.forward(i).addrInvalidSqIdx.value := addrInvalidSqIdx 581 } .otherwise { 582 // may be store inst has been written to sbuffer already. 583 io.forward(i).addrInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx) 584 } 585 io.forward(i).addrInvalid := Mux(RegNext(io.forward(i).uop.loadWaitStrict), RegNext(hasInvalidAddr), addrInvalidFlag) 586 587 // data invalid sq index 588 // make chisel happy 589 val dataInvalidMaskRegWire = Wire(UInt(StoreQueueSize.W)) 590 dataInvalidMaskRegWire := dataInvalidMaskReg 591 val dataInvalidFlag = dataInvalidMaskRegWire.orR 592 593 val dataInvalidSqIdx1 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(dataInvalidMask1Reg)))) 594 val dataInvalidSqIdx2 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(dataInvalidMask2Reg)))) 595 val dataInvalidSqIdx = Mux(dataInvalidMask2Reg.orR, dataInvalidSqIdx2, dataInvalidSqIdx1) 596 597 when (dataInvalidFlag) { 598 io.forward(i).dataInvalidSqIdx.flag := Mux(!s2_differentFlag || dataInvalidSqIdx >= s2_deqPtrExt.value, s2_deqPtrExt.flag, s2_enqPtrExt.flag) 599 io.forward(i).dataInvalidSqIdx.value := dataInvalidSqIdx 600 } .otherwise { 601 // may be store inst has been written to sbuffer already. 602 io.forward(i).dataInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx) 603 } 604 } 605 606 /** 607 * Memory mapped IO / other uncached operations 608 * 609 * States: 610 * (1) writeback from store units: mark as pending 611 * (2) when they reach ROB's head, they can be sent to uncache channel 612 * (3) response from uncache channel: mark as datavalidmask.wen 613 * (4) writeback to ROB (and other units): mark as writebacked 614 * (5) ROB commits the instruction: same as normal instructions 615 */ 616 //(2) when they reach ROB's head, they can be sent to uncache channel 617 val s_idle :: s_req :: s_resp :: s_wb :: s_wait :: Nil = Enum(5) 618 val uncacheState = RegInit(s_idle) 619 switch(uncacheState) { 620 is(s_idle) { 621 when(RegNext(io.rob.pendingst && pending(deqPtr) && allocated(deqPtr) && datavalid(deqPtr) && addrvalid(deqPtr))) { 622 uncacheState := s_req 623 } 624 } 625 is(s_req) { 626 when (io.uncache.req.fire) { 627 when (io.uncacheOutstanding) { 628 uncacheState := s_wb 629 } .otherwise { 630 uncacheState := s_resp 631 } 632 } 633 } 634 is(s_resp) { 635 when(io.uncache.resp.fire) { 636 uncacheState := s_wb 637 } 638 } 639 is(s_wb) { 640 when (io.mmioStout.fire) { 641 uncacheState := s_wait 642 } 643 } 644 is(s_wait) { 645 when(commitCount > 0.U) { 646 uncacheState := s_idle // ready for next mmio 647 } 648 } 649 } 650 io.uncache.req.valid := uncacheState === s_req 651 652 io.uncache.req.bits := DontCare 653 io.uncache.req.bits.cmd := MemoryOpConstants.M_XWR 654 io.uncache.req.bits.addr := paddrModule.io.rdata(0) // data(deqPtr) -> rdata(0) 655 io.uncache.req.bits.data := shiftDataToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).data) 656 io.uncache.req.bits.mask := shiftMaskToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).mask) 657 658 // CBO op type check can be delayed for 1 cycle, 659 // as uncache op will not start in s_idle 660 val cbo_mmio_addr = paddrModule.io.rdata(0) >> 2 << 2 // clear lowest 2 bits for op 661 val cbo_mmio_op = 0.U //TODO 662 val cbo_mmio_data = cbo_mmio_addr | cbo_mmio_op 663 when(RegNext(LSUOpType.isCbo(uop(deqPtr).fuOpType))){ 664 io.uncache.req.bits.addr := DontCare // TODO 665 io.uncache.req.bits.data := paddrModule.io.rdata(0) 666 io.uncache.req.bits.mask := DontCare // TODO 667 } 668 669 io.uncache.req.bits.atomic := atomic(RegNext(rdataPtrExtNext(0)).value) 670 671 when(io.uncache.req.fire){ 672 // mmio store should not be committed until uncache req is sent 673 pending(deqPtr) := false.B 674 675 XSDebug( 676 p"uncache req: pc ${Hexadecimal(uop(deqPtr).pc)} " + 677 p"addr ${Hexadecimal(io.uncache.req.bits.addr)} " + 678 p"data ${Hexadecimal(io.uncache.req.bits.data)} " + 679 p"op ${Hexadecimal(io.uncache.req.bits.cmd)} " + 680 p"mask ${Hexadecimal(io.uncache.req.bits.mask)}\n" 681 ) 682 } 683 684 // (3) response from uncache channel: mark as datavalid 685 io.uncache.resp.ready := true.B 686 687 // (4) writeback to ROB (and other units): mark as writebacked 688 io.mmioStout.valid := uncacheState === s_wb 689 io.mmioStout.bits.uop := uop(deqPtr) 690 io.mmioStout.bits.uop.sqIdx := deqPtrExt(0) 691 io.mmioStout.bits.data := shiftDataToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).data) // dataModule.io.rdata.read(deqPtr) 692 io.mmioStout.bits.debug.isMMIO := true.B 693 io.mmioStout.bits.debug.paddr := DontCare 694 io.mmioStout.bits.debug.isPerfCnt := false.B 695 io.mmioStout.bits.debug.vaddr := DontCare 696 // Remove MMIO inst from store queue after MMIO request is being sent 697 // That inst will be traced by uncache state machine 698 when (io.mmioStout.fire) { 699 allocated(deqPtr) := false.B 700 } 701 702 /** 703 * ROB commits store instructions (mark them as committed) 704 * 705 * (1) When store commits, mark it as committed. 706 * (2) They will not be cancelled and can be sent to lower level. 707 */ 708 XSError(uncacheState =/= s_idle && uncacheState =/= s_wait && commitCount > 0.U, 709 "should not commit instruction when MMIO has not been finished\n") 710 for (i <- 0 until CommitWidth) { 711 when (commitCount > i.U) { // MMIO inst is not in progress 712 if(i == 0){ 713 // MMIO inst should not update committed flag 714 // Note that commit count has been delayed for 1 cycle 715 when(uncacheState === s_idle){ 716 committed(cmtPtrExt(0).value) := true.B 717 } 718 } else { 719 committed(cmtPtrExt(i).value) := true.B 720 } 721 } 722 } 723 cmtPtrExt := cmtPtrExt.map(_ + commitCount) 724 725 // committed stores will not be cancelled and can be sent to lower level. 726 // remove retired insts from sq, add retired store to sbuffer 727 728 // Read data from data module 729 // As store queue grows larger and larger, time needed to read data from data 730 // module keeps growing higher. Now we give data read a whole cycle. 731 732 // Vector stores are written to sbuffer by vector store flow queue rather than sq 733 XSError(io.vecStoreRetire.valid && !vec(rdataPtrExt(0).value), "Vector store flow queue is trying to retire a scalar store") 734 XSError(io.vecStoreRetire.valid && !allocated(rdataPtrExt(0).value), "Vector store flow queue is trying to retire an invalid entry") 735 XSError(io.vecStoreRetire.valid && vec(rdataPtrExt(0).value) && !vecAddrvalid(rdataPtrExt(0).value), "Vector store is trying to retire without write last element!") 736 when (io.vecStoreRetire.valid) { 737 assert(io.vecStoreRetire.bits === rdataPtrExt(0)) 738 vec(rdataPtrExt(0).value) := false.B 739 vecAddrvalid(rdataPtrExt(0).value) := false.B 740 allocated(rdataPtrExt(0).value) := false.B 741 } 742 743 val mmioStall = mmio(rdataPtrExt(0).value) 744 val vecStall = vec(rdataPtrExt(0).value) 745 for (i <- 0 until EnsbufferWidth) { 746 val ptr = rdataPtrExt(i).value 747 dataBuffer.io.enq(i).valid := allocated(ptr) && committed(ptr) && !mmioStall && !vecStall 748 // Note that store data/addr should both be valid after store's commit 749 assert(!dataBuffer.io.enq(i).valid || allvalid(ptr)) 750 dataBuffer.io.enq(i).bits.addr := paddrModule.io.rdata(i) 751 dataBuffer.io.enq(i).bits.vaddr := vaddrModule.io.rdata(i) 752 dataBuffer.io.enq(i).bits.data := dataModule.io.rdata(i).data 753 dataBuffer.io.enq(i).bits.mask := dataModule.io.rdata(i).mask 754 dataBuffer.io.enq(i).bits.wline := paddrModule.io.rlineflag(i) 755 dataBuffer.io.enq(i).bits.sqPtr := rdataPtrExt(i) 756 dataBuffer.io.enq(i).bits.prefetch := prefetch(ptr) 757 } 758 759 // Send data stored in sbufferReqBitsReg to sbuffer 760 for (i <- 0 until EnsbufferWidth) { 761 io.sbuffer(i).valid := dataBuffer.io.deq(i).valid 762 dataBuffer.io.deq(i).ready := io.sbuffer(i).ready 763 // Write line request should have all 1 mask 764 assert(!(io.sbuffer(i).valid && io.sbuffer(i).bits.wline && !io.sbuffer(i).bits.mask.andR)) 765 io.sbuffer(i).bits := DontCare 766 io.sbuffer(i).bits.cmd := MemoryOpConstants.M_XWR 767 io.sbuffer(i).bits.addr := dataBuffer.io.deq(i).bits.addr 768 io.sbuffer(i).bits.vaddr := dataBuffer.io.deq(i).bits.vaddr 769 io.sbuffer(i).bits.data := dataBuffer.io.deq(i).bits.data 770 io.sbuffer(i).bits.mask := dataBuffer.io.deq(i).bits.mask 771 io.sbuffer(i).bits.wline := dataBuffer.io.deq(i).bits.wline 772 io.sbuffer(i).bits.prefetch := dataBuffer.io.deq(i).bits.prefetch 773 774 // io.sbuffer(i).fire is RegNexted, as sbuffer data write takes 2 cycles. 775 // Before data write finish, sbuffer is unable to provide store to load 776 // forward data. As an workaround, deqPtrExt and allocated flag update 777 // is delayed so that load can get the right data from store queue. 778 val ptr = dataBuffer.io.deq(i).bits.sqPtr.value 779 when (RegNext(io.sbuffer(i).fire)) { 780 allocated(RegEnable(ptr, io.sbuffer(i).fire)) := false.B 781 XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr) 782 } 783 } 784 (1 until EnsbufferWidth).foreach(i => when(io.sbuffer(i).fire) { assert(io.sbuffer(i - 1).fire) }) 785 if (coreParams.dcacheParametersOpt.isEmpty) { 786 for (i <- 0 until EnsbufferWidth) { 787 val ptr = deqPtrExt(i).value 788 val ram = DifftestMem(64L * 1024 * 1024 * 1024, 8) 789 val wen = allocated(ptr) && committed(ptr) && !mmio(ptr) 790 val waddr = ((paddrModule.io.rdata(i) - "h80000000".U) >> 3).asUInt 791 val wdata = Mux(paddrModule.io.rdata(i)(3), dataModule.io.rdata(i).data(127, 64), dataModule.io.rdata(i).data(63, 0)) 792 val wmask = Mux(paddrModule.io.rdata(i)(3), dataModule.io.rdata(i).mask(15, 8), dataModule.io.rdata(i).mask(7, 0)) 793 when (wen) { 794 ram.write(waddr, wdata.asTypeOf(Vec(8, UInt(8.W))), wmask.asBools) 795 } 796 } 797 } 798 799 // Read vaddr for mem exception 800 io.exceptionAddr.vaddr := vaddrModule.io.rdata(EnsbufferWidth) 801 io.exceptionAddr.gpaddr := gpaddrModule.io.rdata(EnsbufferWidth) 802 // misprediction recovery / exception redirect 803 // invalidate sq term using robIdx 804 val needCancel = Wire(Vec(StoreQueueSize, Bool())) 805 for (i <- 0 until StoreQueueSize) { 806 needCancel(i) := uop(i).robIdx.needFlush(io.brqRedirect) && allocated(i) && !committed(i) 807 when (needCancel(i)) { 808 allocated(i) := false.B 809 } 810 } 811 812 /** 813* update pointers 814**/ 815 val lastEnqCancel = PopCount(RegNext(VecInit(canEnqueue.zip(enqCancel).map(x => x._1 && x._2)))) // 1 cycle after redirect 816 val lastCycleCancelCount = PopCount(RegNext(needCancel)) // 1 cycle after redirect 817 val lastCycleRedirect = RegNext(io.brqRedirect.valid) // 1 cycle after redirect 818 val enqNumber = Mux(!lastCycleRedirect&&io.enq.canAccept && io.enq.lqCanAccept, PopCount(io.enq.req.map(_.valid)), 0.U) // 1 cycle after redirect 819 820 val lastlastCycleRedirect=RegNext(lastCycleRedirect)// 2 cycle after redirect 821 val redirectCancelCount = RegEnable(lastCycleCancelCount + lastEnqCancel, lastCycleRedirect) // 2 cycle after redirect 822 823 when (lastlastCycleRedirect) { 824 // we recover the pointers in 2 cycle after redirect for better timing 825 enqPtrExt := VecInit(enqPtrExt.map(_ - redirectCancelCount)) 826 }.otherwise { 827 // lastCycleRedirect.valid or nornal case 828 // when lastCycleRedirect.valid, enqNumber === 0.U, enqPtrExt will not change 829 enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) 830 } 831 assert(!(lastCycleRedirect && enqNumber =/= 0.U)) 832 833 deqPtrExt := deqPtrExtNext 834 rdataPtrExt := rdataPtrExtNext 835 836 // val dequeueCount = Mux(io.sbuffer(1).fire, 2.U, Mux(io.sbuffer(0).fire || io.mmioStout.fire, 1.U, 0.U)) 837 838 // If redirect at T0, sqCancelCnt is at T2 839 io.sqCancelCnt := redirectCancelCount 840 val ForceWriteUpper = Wire(UInt(log2Up(StoreQueueSize + 1).W)) 841 ForceWriteUpper := Constantin.createRecord(s"ForceWriteUpper_${p(XSCoreParamsKey).HartId}", initValue = 60) 842 val ForceWriteLower = Wire(UInt(log2Up(StoreQueueSize + 1).W)) 843 ForceWriteLower := Constantin.createRecord(s"ForceWriteLower_${p(XSCoreParamsKey).HartId}", initValue = 55) 844 845 val valid_cnt = PopCount(allocated) 846 io.force_write := RegNext(Mux(valid_cnt >= ForceWriteUpper, true.B, valid_cnt >= ForceWriteLower && io.force_write), init = false.B) 847 848 // io.sqempty will be used by sbuffer 849 // We delay it for 1 cycle for better timing 850 // When sbuffer need to check if it is empty, the pipeline is blocked, which means delay io.sqempty 851 // for 1 cycle will also promise that sq is empty in that cycle 852 io.sqEmpty := RegNext( 853 enqPtrExt(0).value === deqPtrExt(0).value && 854 enqPtrExt(0).flag === deqPtrExt(0).flag 855 ) 856 // perf counter 857 QueuePerf(StoreQueueSize, validCount, !allowEnqueue) 858 io.sqFull := !allowEnqueue 859 XSPerfAccumulate("mmioCycle", uncacheState =/= s_idle) // lq is busy dealing with uncache req 860 XSPerfAccumulate("mmioCnt", io.uncache.req.fire) 861 XSPerfAccumulate("mmio_wb_success", io.mmioStout.fire) 862 XSPerfAccumulate("mmio_wb_blocked", io.mmioStout.valid && !io.mmioStout.ready) 863 XSPerfAccumulate("validEntryCnt", distanceBetween(enqPtrExt(0), deqPtrExt(0))) 864 XSPerfAccumulate("cmtEntryCnt", distanceBetween(cmtPtrExt(0), deqPtrExt(0))) 865 XSPerfAccumulate("nCmtEntryCnt", distanceBetween(enqPtrExt(0), cmtPtrExt(0))) 866 867 val perfValidCount = distanceBetween(enqPtrExt(0), deqPtrExt(0)) 868 val perfEvents = Seq( 869 ("mmioCycle ", uncacheState =/= s_idle), 870 ("mmioCnt ", io.uncache.req.fire), 871 ("mmio_wb_success", io.mmioStout.fire), 872 ("mmio_wb_blocked", io.mmioStout.valid && !io.mmioStout.ready), 873 ("stq_1_4_valid ", (perfValidCount < (StoreQueueSize.U/4.U))), 874 ("stq_2_4_valid ", (perfValidCount > (StoreQueueSize.U/4.U)) & (perfValidCount <= (StoreQueueSize.U/2.U))), 875 ("stq_3_4_valid ", (perfValidCount > (StoreQueueSize.U/2.U)) & (perfValidCount <= (StoreQueueSize.U*3.U/4.U))), 876 ("stq_4_4_valid ", (perfValidCount > (StoreQueueSize.U*3.U/4.U))), 877 ) 878 generatePerfEvent() 879 880 // debug info 881 XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt(0).flag, deqPtr) 882 883 def PrintFlag(flag: Bool, name: String): Unit = { 884 when(flag) { 885 XSDebug(false, true.B, name) 886 }.otherwise { 887 XSDebug(false, true.B, " ") 888 } 889 } 890 891 for (i <- 0 until StoreQueueSize) { 892 XSDebug(i + ": pc %x va %x pa %x data %x ", 893 uop(i).pc, 894 debug_vaddr(i), 895 debug_paddr(i), 896 debug_data(i) 897 ) 898 PrintFlag(allocated(i), "a") 899 PrintFlag(allocated(i) && addrvalid(i), "a") 900 PrintFlag(allocated(i) && datavalid(i), "d") 901 PrintFlag(allocated(i) && committed(i), "c") 902 PrintFlag(allocated(i) && pending(i), "p") 903 PrintFlag(allocated(i) && mmio(i), "m") 904 XSDebug(false, true.B, "\n") 905 } 906 907} 908