1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chisel3._ 20import chisel3.util._ 21import difftest._ 22import difftest.common.DifftestMem 23import org.chipsalliance.cde.config.Parameters 24import utility._ 25import utils._ 26import xiangshan._ 27import xiangshan.cache._ 28import xiangshan.cache.{DCacheLineIO, DCacheWordIO, MemoryOpConstants} 29import xiangshan.backend._ 30import xiangshan.backend.rob.{RobLsqIO, RobPtr} 31import xiangshan.backend.Bundles.{DynInst, MemExuOutput} 32import xiangshan.backend.decode.isa.bitfield.{Riscv32BitInst, XSInstBitFields} 33 34class SqPtr(implicit p: Parameters) extends CircularQueuePtr[SqPtr]( 35 p => p(XSCoreParamsKey).StoreQueueSize 36){ 37} 38 39object SqPtr { 40 def apply(f: Bool, v: UInt)(implicit p: Parameters): SqPtr = { 41 val ptr = Wire(new SqPtr) 42 ptr.flag := f 43 ptr.value := v 44 ptr 45 } 46} 47 48class SqEnqIO(implicit p: Parameters) extends MemBlockBundle { 49 val canAccept = Output(Bool()) 50 val lqCanAccept = Input(Bool()) 51 val needAlloc = Vec(LSQEnqWidth, Input(Bool())) 52 val req = Vec(LSQEnqWidth, Flipped(ValidIO(new DynInst))) 53 val resp = Vec(LSQEnqWidth, Output(new SqPtr)) 54} 55 56class DataBufferEntry (implicit p: Parameters) extends DCacheBundle { 57 val addr = UInt(PAddrBits.W) 58 val vaddr = UInt(VAddrBits.W) 59 val data = UInt(VLEN.W) 60 val mask = UInt((VLEN/8).W) 61 val wline = Bool() 62 val sqPtr = new SqPtr 63 val prefetch = Bool() 64} 65 66// Store Queue 67class StoreQueue(implicit p: Parameters) extends XSModule 68 with HasDCacheParameters with HasCircularQueuePtrHelper with HasPerfEvents { 69 val io = IO(new Bundle() { 70 val hartId = Input(UInt(hartIdLen.W)) 71 val enq = new SqEnqIO 72 val brqRedirect = Flipped(ValidIO(new Redirect)) 73 val storeAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // store addr, data is not included 74 val vecStoreAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // vec store addr, data is not include , from vsFlowQueue 75 val storeAddrInRe = Vec(StorePipelineWidth, Input(new LsPipelineBundle())) // store more mmio and exception 76 val storeDataIn = Vec(StorePipelineWidth, Flipped(Valid(new MemExuOutput))) // store data, send to sq from rs 77 val storeMaskIn = Vec(StorePipelineWidth, Flipped(Valid(new StoreMaskBundle))) // store mask, send to sq from rs 78 val sbuffer = Vec(EnsbufferWidth, Decoupled(new DCacheWordReqWithVaddrAndPfFlag)) // write committed store to sbuffer 79 val uncacheOutstanding = Input(Bool()) 80 val mmioStout = DecoupledIO(new MemExuOutput) // writeback uncached store 81 val forward = Vec(LoadPipelineWidth, Flipped(new PipeLoadForwardQueryIO)) 82 val rob = Flipped(new RobLsqIO) 83 val uncache = new UncacheWordIO 84 // val refill = Flipped(Valid(new DCacheLineReq )) 85 val exceptionAddr = new ExceptionAddrIO 86 val sqEmpty = Output(Bool()) 87 val stAddrReadySqPtr = Output(new SqPtr) 88 val stAddrReadyVec = Output(Vec(StoreQueueSize, Bool())) 89 val stDataReadySqPtr = Output(new SqPtr) 90 val stDataReadyVec = Output(Vec(StoreQueueSize, Bool())) 91 val stIssuePtr = Output(new SqPtr) 92 val sqDeqPtr = Output(new SqPtr) 93 val sqFull = Output(Bool()) 94 val sqCancelCnt = Output(UInt(log2Up(StoreQueueSize + 1).W)) 95 val sqDeq = Output(UInt(log2Ceil(EnsbufferWidth + 1).W)) 96 val force_write = Output(Bool()) 97 val vecStoreRetire = Flipped(ValidIO(new SqPtr)) 98 }) 99 100 println("StoreQueue: size:" + StoreQueueSize) 101 102 // data modules 103 val uop = Reg(Vec(StoreQueueSize, new DynInst)) 104 // val data = Reg(Vec(StoreQueueSize, new LsqEntry)) 105 val dataModule = Module(new SQDataModule( 106 numEntries = StoreQueueSize, 107 numRead = EnsbufferWidth, 108 numWrite = StorePipelineWidth, 109 numForward = LoadPipelineWidth 110 )) 111 dataModule.io := DontCare 112 val paddrModule = Module(new SQAddrModule( 113 dataWidth = PAddrBits, 114 numEntries = StoreQueueSize, 115 numRead = EnsbufferWidth, 116 numWrite = StorePipelineWidth, 117 numForward = LoadPipelineWidth 118 )) 119 paddrModule.io := DontCare 120 val vaddrModule = Module(new SQAddrModule( 121 dataWidth = VAddrBits, 122 numEntries = StoreQueueSize, 123 numRead = EnsbufferWidth + 1, // sbuffer + badvaddr 1 (TODO) 124 numWrite = StorePipelineWidth, 125 numForward = LoadPipelineWidth 126 )) 127 val gpaddrModule = Module(new SQAddrModule( 128 dataWidth = GPAddrBits, 129 numEntries = StoreQueueSize, 130 numRead = EnsbufferWidth + 1, 131 numWrite = StorePipelineWidth, 132 numForward = LoadPipelineWidth 133 )) 134 vaddrModule.io := DontCare 135 gpaddrModule.io := DontCare 136 val dataBuffer = Module(new DatamoduleResultBuffer(new DataBufferEntry)) 137 val debug_paddr = Reg(Vec(StoreQueueSize, UInt((PAddrBits).W))) 138 val debug_vaddr = Reg(Vec(StoreQueueSize, UInt((VAddrBits).W))) 139 val debug_data = Reg(Vec(StoreQueueSize, UInt((XLEN).W))) 140 141 // state & misc 142 val allocated = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // sq entry has been allocated 143 val addrvalid = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // non-mmio addr is valid 144 val datavalid = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // non-mmio data is valid 145 val allvalid = VecInit((0 until StoreQueueSize).map(i => addrvalid(i) && datavalid(i))) // non-mmio data & addr is valid 146 val committed = Reg(Vec(StoreQueueSize, Bool())) // inst has been committed by rob 147 val pending = Reg(Vec(StoreQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of rob 148 val mmio = Reg(Vec(StoreQueueSize, Bool())) // mmio: inst is an mmio inst 149 val atomic = Reg(Vec(StoreQueueSize, Bool())) 150 val prefetch = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // need prefetch when committing this store to sbuffer? 151 val vec = Reg(Vec(StoreQueueSize, Bool())) 152 val vecAddrvalid = Reg(Vec(StoreQueueSize, Bool())) // TODO 153 154 // ptr 155 val enqPtrExt = RegInit(VecInit((0 until io.enq.req.length).map(_.U.asTypeOf(new SqPtr)))) 156 val rdataPtrExt = RegInit(VecInit((0 until EnsbufferWidth).map(_.U.asTypeOf(new SqPtr)))) 157 val deqPtrExt = RegInit(VecInit((0 until EnsbufferWidth).map(_.U.asTypeOf(new SqPtr)))) 158 val cmtPtrExt = RegInit(VecInit((0 until CommitWidth).map(_.U.asTypeOf(new SqPtr)))) 159 val addrReadyPtrExt = RegInit(0.U.asTypeOf(new SqPtr)) 160 val dataReadyPtrExt = RegInit(0.U.asTypeOf(new SqPtr)) 161 val validCounter = RegInit(0.U(log2Ceil(VirtualLoadQueueSize + 1).W)) 162 163 val enqPtr = enqPtrExt(0).value 164 val deqPtr = deqPtrExt(0).value 165 val cmtPtr = cmtPtrExt(0).value 166 167 val validCount = distanceBetween(enqPtrExt(0), deqPtrExt(0)) 168 val allowEnqueue = validCount <= (StoreQueueSize - LSQStEnqWidth).U 169 170 val deqMask = UIntToMask(deqPtr, StoreQueueSize) 171 val enqMask = UIntToMask(enqPtr, StoreQueueSize) 172 173 val commitCount = RegNext(io.rob.scommit) 174 175 // store can be committed by ROB 176 io.rob.mmio := DontCare 177 io.rob.uop := DontCare 178 179 // Read dataModule 180 assert(EnsbufferWidth <= 2) 181 // rdataPtrExtNext and rdataPtrExtNext+1 entry will be read from dataModule 182 val rdataPtrExtNext = WireInit(Mux(dataBuffer.io.enq(1).fire, 183 VecInit(rdataPtrExt.map(_ + 2.U)), 184 Mux(dataBuffer.io.enq(0).fire || io.mmioStout.fire || io.vecStoreRetire.valid, 185 VecInit(rdataPtrExt.map(_ + 1.U)), 186 rdataPtrExt 187 ) 188 )) 189 190 // deqPtrExtNext traces which inst is about to leave store queue 191 // 192 // io.sbuffer(i).fire is RegNexted, as sbuffer data write takes 2 cycles. 193 // Before data write finish, sbuffer is unable to provide store to load 194 // forward data. As an workaround, deqPtrExt and allocated flag update 195 // is delayed so that load can get the right data from store queue. 196 // 197 // Modify deqPtrExtNext and io.sqDeq with care! 198 val deqPtrExtNext = Mux(RegNext(io.sbuffer(1).fire), 199 VecInit(deqPtrExt.map(_ + 2.U)), 200 Mux(RegNext(io.sbuffer(0).fire) || io.mmioStout.fire || io.vecStoreRetire.valid, 201 VecInit(deqPtrExt.map(_ + 1.U)), 202 deqPtrExt 203 ) 204 ) 205 io.sqDeq := RegNext(Mux(RegNext(io.sbuffer(1).fire), 2.U, 206 Mux(RegNext(io.sbuffer(0).fire) || io.mmioStout.fire || io.vecStoreRetire.valid, 1.U, 0.U) 207 )) 208 assert(!RegNext(RegNext(io.sbuffer(0).fire) && io.mmioStout.fire)) 209 210 for (i <- 0 until EnsbufferWidth) { 211 dataModule.io.raddr(i) := rdataPtrExtNext(i).value 212 paddrModule.io.raddr(i) := rdataPtrExtNext(i).value 213 vaddrModule.io.raddr(i) := rdataPtrExtNext(i).value 214 gpaddrModule.io.raddr(i) := rdataPtrExtNext(i).value 215 } 216 217 // no inst will be committed 1 cycle before tval update 218 vaddrModule.io.raddr(EnsbufferWidth) := (cmtPtrExt(0) + commitCount).value 219 gpaddrModule.io.raddr(EnsbufferWidth) := (cmtPtrExt(0) + commitCount).value 220 /** 221 * Enqueue at dispatch 222 * 223 * Currently, StoreQueue only allows enqueue when #emptyEntries > EnqWidth 224 */ 225 io.enq.canAccept := allowEnqueue 226 val canEnqueue = io.enq.req.map(_.valid) 227 val enqCancel = io.enq.req.map(_.bits.robIdx.needFlush(io.brqRedirect)) 228 for (i <- 0 until io.enq.req.length) { 229 val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i)) 230 val sqIdx = enqPtrExt(offset) 231 val index = io.enq.req(i).bits.sqIdx.value 232 val enqInstr = io.enq.req(i).bits.instr.asTypeOf(new XSInstBitFields) 233 when (canEnqueue(i) && !enqCancel(i)) { 234 uop(index) := io.enq.req(i).bits 235 // NOTE: the index will be used when replay 236 uop(index).sqIdx := sqIdx 237 allocated(index) := true.B 238 datavalid(index) := false.B 239 addrvalid(index) := false.B 240 committed(index) := false.B 241 pending(index) := false.B 242 prefetch(index) := false.B 243 mmio(index) := false.B 244 vec(index) := enqInstr.isVecStore // check vector store by the encoding of inst 245 vecAddrvalid(index) := false.B//TODO 246 247 XSError(!io.enq.canAccept || !io.enq.lqCanAccept, s"must accept $i\n") 248 XSError(index =/= sqIdx.value, s"must be the same entry $i\n") 249 } 250 io.enq.resp(i) := sqIdx 251 } 252 XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n") 253 254 /** 255 * Update addr/dataReadyPtr when issue from rs 256 */ 257 // update issuePtr 258 val IssuePtrMoveStride = 4 259 require(IssuePtrMoveStride >= 2) 260 261 val addrReadyLookupVec = (0 until IssuePtrMoveStride).map(addrReadyPtrExt + _.U) 262 val addrReadyLookup = addrReadyLookupVec.map(ptr => allocated(ptr.value) && (mmio(ptr.value) || addrvalid(ptr.value) || (vec(ptr.value) && vecAddrvalid(ptr.value))) && ptr =/= enqPtrExt(0)) 263 val nextAddrReadyPtr = addrReadyPtrExt + PriorityEncoder(VecInit(addrReadyLookup.map(!_) :+ true.B)) 264 addrReadyPtrExt := nextAddrReadyPtr 265 266 (0 until StoreQueueSize).map(i => { 267 io.stAddrReadyVec(i) := RegNext(allocated(i) && (mmio(i) || addrvalid(i))) 268 }) 269 270 when (io.brqRedirect.valid) { 271 addrReadyPtrExt := Mux( 272 isAfter(cmtPtrExt(0), deqPtrExt(0)), 273 cmtPtrExt(0), 274 deqPtrExtNext(0) // for mmio insts, deqPtr may be ahead of cmtPtr 275 ) 276 } 277 278 io.stAddrReadySqPtr := addrReadyPtrExt 279 280 // update 281 val dataReadyLookupVec = (0 until IssuePtrMoveStride).map(dataReadyPtrExt + _.U) 282 val dataReadyLookup = dataReadyLookupVec.map(ptr => allocated(ptr.value) && (mmio(ptr.value) || datavalid(ptr.value) || vec(ptr.value)) && ptr =/= enqPtrExt(0)) // TODO : flag of vector store data valid not add yet 283 val nextDataReadyPtr = dataReadyPtrExt + PriorityEncoder(VecInit(dataReadyLookup.map(!_) :+ true.B)) 284 dataReadyPtrExt := nextDataReadyPtr 285 286 (0 until StoreQueueSize).map(i => { 287 io.stDataReadyVec(i) := RegNext(allocated(i) && (mmio(i) || datavalid(i))) 288 }) 289 290 when (io.brqRedirect.valid) { 291 dataReadyPtrExt := Mux( 292 isAfter(cmtPtrExt(0), deqPtrExt(0)), 293 cmtPtrExt(0), 294 deqPtrExtNext(0) // for mmio insts, deqPtr may be ahead of cmtPtr 295 ) 296 } 297 298 io.stDataReadySqPtr := dataReadyPtrExt 299 io.stIssuePtr := enqPtrExt(0) 300 io.sqDeqPtr := deqPtrExt(0) 301 302 /** 303 * Writeback store from store units 304 * 305 * Most store instructions writeback to regfile in the previous cycle. 306 * However, 307 * (1) For an mmio instruction with exceptions, we need to mark it as addrvalid 308 * (in this way it will trigger an exception when it reaches ROB's head) 309 * instead of pending to avoid sending them to lower level. 310 * (2) For an mmio instruction without exceptions, we mark it as pending. 311 * When the instruction reaches ROB's head, StoreQueue sends it to uncache channel. 312 * Upon receiving the response, StoreQueue writes back the instruction 313 * through arbiter with store units. It will later commit as normal. 314 */ 315 316 // Write addr to sq 317 for (i <- 0 until StorePipelineWidth) { 318 paddrModule.io.wen(i) := false.B 319 vaddrModule.io.wen(i) := false.B 320 gpaddrModule.io.wen(i) := false.B 321 dataModule.io.mask.wen(i) := false.B 322 val stWbIndex = io.storeAddrIn(i).bits.uop.sqIdx.value 323 when (io.storeAddrIn(i).fire) { 324 val addr_valid = !io.storeAddrIn(i).bits.miss 325 addrvalid(stWbIndex) := addr_valid //!io.storeAddrIn(i).bits.mmio 326 // pending(stWbIndex) := io.storeAddrIn(i).bits.mmio 327 328 paddrModule.io.waddr(i) := stWbIndex 329 paddrModule.io.wdata(i) := io.storeAddrIn(i).bits.paddr 330 paddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 331 paddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 332 paddrModule.io.wen(i) := true.B 333 334 vaddrModule.io.waddr(i) := stWbIndex 335 vaddrModule.io.wdata(i) := io.storeAddrIn(i).bits.vaddr 336 vaddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 337 vaddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 338 vaddrModule.io.wen(i) := true.B 339 340 gpaddrModule.io.waddr(i) := stWbIndex 341 gpaddrModule.io.wdata(i) := io.storeAddrIn(i).bits.gpaddr 342 gpaddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 343 gpaddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 344 gpaddrModule.io.wen(i) := true.B 345 346 debug_paddr(paddrModule.io.waddr(i)) := paddrModule.io.wdata(i) 347 348 // mmio(stWbIndex) := io.storeAddrIn(i).bits.mmio 349 350 uop(stWbIndex) := io.storeAddrIn(i).bits.uop 351 uop(stWbIndex).debugInfo := io.storeAddrIn(i).bits.uop.debugInfo 352 XSInfo("store addr write to sq idx %d pc 0x%x miss:%d vaddr %x paddr %x mmio %x\n", 353 io.storeAddrIn(i).bits.uop.sqIdx.value, 354 io.storeAddrIn(i).bits.uop.pc, 355 io.storeAddrIn(i).bits.miss, 356 io.storeAddrIn(i).bits.vaddr, 357 io.storeAddrIn(i).bits.paddr, 358 io.storeAddrIn(i).bits.mmio 359 ) 360 } 361 362 // re-replinish mmio, for pma/pmp will get mmio one cycle later 363 val storeAddrInFireReg = RegNext(io.storeAddrIn(i).fire && !io.storeAddrIn(i).bits.miss) 364 val stWbIndexReg = RegNext(stWbIndex) 365 when (storeAddrInFireReg) { 366 pending(stWbIndexReg) := io.storeAddrInRe(i).mmio 367 mmio(stWbIndexReg) := io.storeAddrInRe(i).mmio 368 atomic(stWbIndexReg) := io.storeAddrInRe(i).atomic 369 } 370 // dcache miss info (one cycle later than storeIn) 371 // if dcache report a miss in sta pipeline, this store will trigger a prefetch when committing to sbuffer (if EnableAtCommitMissTrigger) 372 when (storeAddrInFireReg) { 373 prefetch(stWbIndexReg) := io.storeAddrInRe(i).miss 374 } 375 376 when(vaddrModule.io.wen(i)){ 377 debug_vaddr(vaddrModule.io.waddr(i)) := vaddrModule.io.wdata(i) 378 } 379 // TODO : When lastElem issue to stu or inactivative issue in vsFlowQueue, set vector store addr ready 380 val vecStWbIndex = io.vecStoreAddrIn(i).bits.uop.sqIdx.value 381 when(io.vecStoreAddrIn(i).fire){ 382 vecAddrvalid(vecStWbIndex) := true.B 383 } 384 } 385 386 // Write data to sq 387 // Now store data pipeline is actually 2 stages 388 for (i <- 0 until StorePipelineWidth) { 389 dataModule.io.data.wen(i) := false.B 390 val stWbIndex = io.storeDataIn(i).bits.uop.sqIdx.value 391 // sq data write takes 2 cycles: 392 // sq data write s0 393 when (io.storeDataIn(i).fire) { 394 // send data write req to data module 395 dataModule.io.data.waddr(i) := stWbIndex 396 dataModule.io.data.wdata(i) := Mux(io.storeDataIn(i).bits.uop.fuOpType === LSUOpType.cbo_zero, 397 0.U, 398 genWdata(io.storeDataIn(i).bits.data, io.storeDataIn(i).bits.uop.fuOpType(1,0)) 399 ) 400 dataModule.io.data.wen(i) := true.B 401 402 debug_data(dataModule.io.data.waddr(i)) := dataModule.io.data.wdata(i) 403 404 XSInfo("store data write to sq idx %d pc 0x%x data %x -> %x\n", 405 io.storeDataIn(i).bits.uop.sqIdx.value, 406 io.storeDataIn(i).bits.uop.pc, 407 io.storeDataIn(i).bits.data, 408 dataModule.io.data.wdata(i) 409 ) 410 } 411 // sq data write s1 412 when ( 413 RegNext(io.storeDataIn(i).fire) 414 // && !RegNext(io.storeDataIn(i).bits.uop).robIdx.needFlush(io.brqRedirect) 415 ) { 416 datavalid(RegNext(stWbIndex)) := true.B 417 } 418 } 419 420 // Write mask to sq 421 for (i <- 0 until StorePipelineWidth) { 422 // sq mask write s0 423 when (io.storeMaskIn(i).fire) { 424 // send data write req to data module 425 dataModule.io.mask.waddr(i) := io.storeMaskIn(i).bits.sqIdx.value 426 dataModule.io.mask.wdata(i) := io.storeMaskIn(i).bits.mask 427 dataModule.io.mask.wen(i) := true.B 428 } 429 } 430 431 /** 432 * load forward query 433 * 434 * Check store queue for instructions that is older than the load. 435 * The response will be valid at the next cycle after req. 436 */ 437 // check over all lq entries and forward data from the first matched store 438 for (i <- 0 until LoadPipelineWidth) { 439 // Compare deqPtr (deqPtr) and forward.sqIdx, we have two cases: 440 // (1) if they have the same flag, we need to check range(tail, sqIdx) 441 // (2) if they have different flags, we need to check range(tail, VirtualLoadQueueSize) and range(0, sqIdx) 442 // Forward1: Mux(same_flag, range(tail, sqIdx), range(tail, VirtualLoadQueueSize)) 443 // Forward2: Mux(same_flag, 0.U, range(0, sqIdx) ) 444 // i.e. forward1 is the target entries with the same flag bits and forward2 otherwise 445 val differentFlag = deqPtrExt(0).flag =/= io.forward(i).sqIdx.flag 446 val forwardMask = io.forward(i).sqIdxMask 447 // all addrvalid terms need to be checked 448 val addrValidVec = WireInit(VecInit((0 until StoreQueueSize).map(j => addrvalid(j) && allocated(j)))) 449 val dataValidVec = WireInit(VecInit((0 until StoreQueueSize).map(j => datavalid(j)))) 450 val allValidVec = WireInit(VecInit((0 until StoreQueueSize).map(j => addrvalid(j) && datavalid(j) && allocated(j)))) 451 452 val lfstEnable = Constantin.createRecord("LFSTEnable", LFSTEnable.B).orR 453 val storeSetHitVec = Mux(lfstEnable, 454 WireInit(VecInit((0 until StoreQueueSize).map(j => io.forward(i).uop.loadWaitBit && uop(j).robIdx === io.forward(i).uop.waitForRobIdx))), 455 WireInit(VecInit((0 until StoreQueueSize).map(j => uop(j).storeSetHit && uop(j).ssid === io.forward(i).uop.ssid))) 456 ) 457 458 val forwardMask1 = Mux(differentFlag, ~deqMask, deqMask ^ forwardMask) 459 val forwardMask2 = Mux(differentFlag, forwardMask, 0.U(StoreQueueSize.W)) 460 val canForward1 = forwardMask1 & allValidVec.asUInt 461 val canForward2 = forwardMask2 & allValidVec.asUInt 462 val needForward = Mux(differentFlag, ~deqMask | forwardMask, deqMask ^ forwardMask) 463 464 XSDebug(p"$i f1 ${Binary(canForward1)} f2 ${Binary(canForward2)} " + 465 p"sqIdx ${io.forward(i).sqIdx} pa ${Hexadecimal(io.forward(i).paddr)}\n" 466 ) 467 468 // do real fwd query (cam lookup in load_s1) 469 dataModule.io.needForward(i)(0) := canForward1 & vaddrModule.io.forwardMmask(i).asUInt 470 dataModule.io.needForward(i)(1) := canForward2 & vaddrModule.io.forwardMmask(i).asUInt 471 472 vaddrModule.io.forwardMdata(i) := io.forward(i).vaddr 473 vaddrModule.io.forwardDataMask(i) := io.forward(i).mask 474 paddrModule.io.forwardMdata(i) := io.forward(i).paddr 475 paddrModule.io.forwardDataMask(i) := io.forward(i).mask 476 gpaddrModule.io.forwardMdata(i) := io.forward(i).gpaddr 477 gpaddrModule.io.forwardDataMask(i) := io.forward(i).mask 478 479 480 // vaddr cam result does not equal to paddr cam result 481 // replay needed 482 // val vpmaskNotEqual = ((paddrModule.io.forwardMmask(i).asUInt ^ vaddrModule.io.forwardMmask(i).asUInt) & needForward) =/= 0.U 483 // val vaddrMatchFailed = vpmaskNotEqual && io.forward(i).valid 484 val vpmaskNotEqual = ( 485 (RegNext(paddrModule.io.forwardMmask(i).asUInt) ^ RegNext(vaddrModule.io.forwardMmask(i).asUInt)) & 486 RegNext(needForward) & 487 RegNext(addrValidVec.asUInt) 488 ) =/= 0.U 489 val vaddrMatchFailed = vpmaskNotEqual && RegNext(io.forward(i).valid) 490 when (vaddrMatchFailed) { 491 XSInfo("vaddrMatchFailed: pc %x pmask %x vmask %x\n", 492 RegNext(io.forward(i).uop.pc), 493 RegNext(needForward & paddrModule.io.forwardMmask(i).asUInt), 494 RegNext(needForward & vaddrModule.io.forwardMmask(i).asUInt) 495 ); 496 } 497 XSPerfAccumulate("vaddr_match_failed", vpmaskNotEqual) 498 XSPerfAccumulate("vaddr_match_really_failed", vaddrMatchFailed) 499 500 // Fast forward mask will be generated immediately (load_s1) 501 io.forward(i).forwardMaskFast := dataModule.io.forwardMaskFast(i) 502 503 // Forward result will be generated 1 cycle later (load_s2) 504 io.forward(i).forwardMask := dataModule.io.forwardMask(i) 505 io.forward(i).forwardData := dataModule.io.forwardData(i) 506 // If addr match, data not ready, mark it as dataInvalid 507 // load_s1: generate dataInvalid in load_s1 to set fastUop 508 val dataInvalidMask1 = (addrValidVec.asUInt & ~dataValidVec.asUInt & vaddrModule.io.forwardMmask(i).asUInt & forwardMask1.asUInt) 509 val dataInvalidMask2 = (addrValidVec.asUInt & ~dataValidVec.asUInt & vaddrModule.io.forwardMmask(i).asUInt & forwardMask2.asUInt) 510 val dataInvalidMask = dataInvalidMask1 | dataInvalidMask2 511 io.forward(i).dataInvalidFast := dataInvalidMask.orR 512 513 // make chisel happy 514 val dataInvalidMask1Reg = Wire(UInt(StoreQueueSize.W)) 515 dataInvalidMask1Reg := RegNext(dataInvalidMask1) 516 // make chisel happy 517 val dataInvalidMask2Reg = Wire(UInt(StoreQueueSize.W)) 518 dataInvalidMask2Reg := RegNext(dataInvalidMask2) 519 val dataInvalidMaskReg = dataInvalidMask1Reg | dataInvalidMask2Reg 520 521 // If SSID match, address not ready, mark it as addrInvalid 522 // load_s2: generate addrInvalid 523 val addrInvalidMask1 = (~addrValidVec.asUInt & storeSetHitVec.asUInt & forwardMask1.asUInt) 524 val addrInvalidMask2 = (~addrValidVec.asUInt & storeSetHitVec.asUInt & forwardMask2.asUInt) 525 // make chisel happy 526 val addrInvalidMask1Reg = Wire(UInt(StoreQueueSize.W)) 527 addrInvalidMask1Reg := RegNext(addrInvalidMask1) 528 // make chisel happy 529 val addrInvalidMask2Reg = Wire(UInt(StoreQueueSize.W)) 530 addrInvalidMask2Reg := RegNext(addrInvalidMask2) 531 val addrInvalidMaskReg = addrInvalidMask1Reg | addrInvalidMask2Reg 532 533 // load_s2 534 io.forward(i).dataInvalid := RegNext(io.forward(i).dataInvalidFast) 535 // check if vaddr forward mismatched 536 io.forward(i).matchInvalid := vaddrMatchFailed 537 538 // data invalid sq index 539 // check whether false fail 540 // check flag 541 val s2_differentFlag = RegNext(differentFlag) 542 val s2_enqPtrExt = RegNext(enqPtrExt(0)) 543 val s2_deqPtrExt = RegNext(deqPtrExt(0)) 544 545 // addr invalid sq index 546 // make chisel happy 547 val addrInvalidMaskRegWire = Wire(UInt(StoreQueueSize.W)) 548 addrInvalidMaskRegWire := addrInvalidMaskReg 549 val addrInvalidFlag = addrInvalidMaskRegWire.orR 550 val hasInvalidAddr = (~addrValidVec.asUInt & needForward).orR 551 552 val addrInvalidSqIdx1 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(addrInvalidMask1Reg)))) 553 val addrInvalidSqIdx2 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(addrInvalidMask2Reg)))) 554 val addrInvalidSqIdx = Mux(addrInvalidMask2Reg.orR, addrInvalidSqIdx2, addrInvalidSqIdx1) 555 556 // store-set content management 557 // +-----------------------+ 558 // | Search a SSID for the | 559 // | load operation | 560 // +-----------------------+ 561 // | 562 // V 563 // +-------------------+ 564 // | load wait strict? | 565 // +-------------------+ 566 // | 567 // V 568 // +----------------------+ 569 // Set| |Clean 570 // V V 571 // +------------------------+ +------------------------------+ 572 // | Waiting for all older | | Wait until the corresponding | 573 // | stores operations | | older store operations | 574 // +------------------------+ +------------------------------+ 575 576 577 578 when (RegNext(io.forward(i).uop.loadWaitStrict)) { 579 io.forward(i).addrInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx - 1.U) 580 } .elsewhen (addrInvalidFlag) { 581 io.forward(i).addrInvalidSqIdx.flag := Mux(!s2_differentFlag || addrInvalidSqIdx >= s2_deqPtrExt.value, s2_deqPtrExt.flag, s2_enqPtrExt.flag) 582 io.forward(i).addrInvalidSqIdx.value := addrInvalidSqIdx 583 } .otherwise { 584 // may be store inst has been written to sbuffer already. 585 io.forward(i).addrInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx) 586 } 587 io.forward(i).addrInvalid := Mux(RegNext(io.forward(i).uop.loadWaitStrict), RegNext(hasInvalidAddr), addrInvalidFlag) 588 589 // data invalid sq index 590 // make chisel happy 591 val dataInvalidMaskRegWire = Wire(UInt(StoreQueueSize.W)) 592 dataInvalidMaskRegWire := dataInvalidMaskReg 593 val dataInvalidFlag = dataInvalidMaskRegWire.orR 594 595 val dataInvalidSqIdx1 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(dataInvalidMask1Reg)))) 596 val dataInvalidSqIdx2 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(dataInvalidMask2Reg)))) 597 val dataInvalidSqIdx = Mux(dataInvalidMask2Reg.orR, dataInvalidSqIdx2, dataInvalidSqIdx1) 598 599 when (dataInvalidFlag) { 600 io.forward(i).dataInvalidSqIdx.flag := Mux(!s2_differentFlag || dataInvalidSqIdx >= s2_deqPtrExt.value, s2_deqPtrExt.flag, s2_enqPtrExt.flag) 601 io.forward(i).dataInvalidSqIdx.value := dataInvalidSqIdx 602 } .otherwise { 603 // may be store inst has been written to sbuffer already. 604 io.forward(i).dataInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx) 605 } 606 } 607 608 /** 609 * Memory mapped IO / other uncached operations 610 * 611 * States: 612 * (1) writeback from store units: mark as pending 613 * (2) when they reach ROB's head, they can be sent to uncache channel 614 * (3) response from uncache channel: mark as datavalidmask.wen 615 * (4) writeback to ROB (and other units): mark as writebacked 616 * (5) ROB commits the instruction: same as normal instructions 617 */ 618 //(2) when they reach ROB's head, they can be sent to uncache channel 619 val s_idle :: s_req :: s_resp :: s_wb :: s_wait :: Nil = Enum(5) 620 val uncacheState = RegInit(s_idle) 621 switch(uncacheState) { 622 is(s_idle) { 623 when(RegNext(io.rob.pendingst && pending(deqPtr) && allocated(deqPtr) && datavalid(deqPtr) && addrvalid(deqPtr))) { 624 uncacheState := s_req 625 } 626 } 627 is(s_req) { 628 when (io.uncache.req.fire) { 629 when (io.uncacheOutstanding) { 630 uncacheState := s_wb 631 } .otherwise { 632 uncacheState := s_resp 633 } 634 } 635 } 636 is(s_resp) { 637 when(io.uncache.resp.fire) { 638 uncacheState := s_wb 639 } 640 } 641 is(s_wb) { 642 when (io.mmioStout.fire) { 643 uncacheState := s_wait 644 } 645 } 646 is(s_wait) { 647 when(commitCount > 0.U) { 648 uncacheState := s_idle // ready for next mmio 649 } 650 } 651 } 652 io.uncache.req.valid := uncacheState === s_req 653 654 io.uncache.req.bits := DontCare 655 io.uncache.req.bits.cmd := MemoryOpConstants.M_XWR 656 io.uncache.req.bits.addr := paddrModule.io.rdata(0) // data(deqPtr) -> rdata(0) 657 io.uncache.req.bits.data := shiftDataToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).data) 658 io.uncache.req.bits.mask := shiftMaskToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).mask) 659 660 // CBO op type check can be delayed for 1 cycle, 661 // as uncache op will not start in s_idle 662 val cbo_mmio_addr = paddrModule.io.rdata(0) >> 2 << 2 // clear lowest 2 bits for op 663 val cbo_mmio_op = 0.U //TODO 664 val cbo_mmio_data = cbo_mmio_addr | cbo_mmio_op 665 when(RegNext(LSUOpType.isCbo(uop(deqPtr).fuOpType))){ 666 io.uncache.req.bits.addr := DontCare // TODO 667 io.uncache.req.bits.data := paddrModule.io.rdata(0) 668 io.uncache.req.bits.mask := DontCare // TODO 669 } 670 671 io.uncache.req.bits.atomic := atomic(RegNext(rdataPtrExtNext(0)).value) 672 673 when(io.uncache.req.fire){ 674 // mmio store should not be committed until uncache req is sent 675 pending(deqPtr) := false.B 676 677 XSDebug( 678 p"uncache req: pc ${Hexadecimal(uop(deqPtr).pc)} " + 679 p"addr ${Hexadecimal(io.uncache.req.bits.addr)} " + 680 p"data ${Hexadecimal(io.uncache.req.bits.data)} " + 681 p"op ${Hexadecimal(io.uncache.req.bits.cmd)} " + 682 p"mask ${Hexadecimal(io.uncache.req.bits.mask)}\n" 683 ) 684 } 685 686 // (3) response from uncache channel: mark as datavalid 687 io.uncache.resp.ready := true.B 688 689 // (4) writeback to ROB (and other units): mark as writebacked 690 io.mmioStout.valid := uncacheState === s_wb 691 io.mmioStout.bits.uop := uop(deqPtr) 692 io.mmioStout.bits.uop.sqIdx := deqPtrExt(0) 693 io.mmioStout.bits.data := shiftDataToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).data) // dataModule.io.rdata.read(deqPtr) 694 io.mmioStout.bits.debug.isMMIO := true.B 695 io.mmioStout.bits.debug.paddr := DontCare 696 io.mmioStout.bits.debug.isPerfCnt := false.B 697 io.mmioStout.bits.debug.vaddr := DontCare 698 // Remove MMIO inst from store queue after MMIO request is being sent 699 // That inst will be traced by uncache state machine 700 when (io.mmioStout.fire) { 701 allocated(deqPtr) := false.B 702 } 703 704 /** 705 * ROB commits store instructions (mark them as committed) 706 * 707 * (1) When store commits, mark it as committed. 708 * (2) They will not be cancelled and can be sent to lower level. 709 */ 710 XSError(uncacheState =/= s_idle && uncacheState =/= s_wait && commitCount > 0.U, 711 "should not commit instruction when MMIO has not been finished\n") 712 for (i <- 0 until CommitWidth) { 713 when (commitCount > i.U) { // MMIO inst is not in progress 714 if(i == 0){ 715 // MMIO inst should not update committed flag 716 // Note that commit count has been delayed for 1 cycle 717 when(uncacheState === s_idle){ 718 committed(cmtPtrExt(0).value) := true.B 719 } 720 } else { 721 committed(cmtPtrExt(i).value) := true.B 722 } 723 } 724 } 725 cmtPtrExt := cmtPtrExt.map(_ + commitCount) 726 727 // committed stores will not be cancelled and can be sent to lower level. 728 // remove retired insts from sq, add retired store to sbuffer 729 730 // Read data from data module 731 // As store queue grows larger and larger, time needed to read data from data 732 // module keeps growing higher. Now we give data read a whole cycle. 733 734 // Vector stores are written to sbuffer by vector store flow queue rather than sq 735 XSError(io.vecStoreRetire.valid && !vec(rdataPtrExt(0).value), "Vector store flow queue is trying to retire a scalar store") 736 XSError(io.vecStoreRetire.valid && !allocated(rdataPtrExt(0).value), "Vector store flow queue is trying to retire an invalid entry") 737 XSError(io.vecStoreRetire.valid && vec(rdataPtrExt(0).value) && !vecAddrvalid(rdataPtrExt(0).value), "Vector store is trying to retire without write last element!") 738 when (io.vecStoreRetire.valid) { 739 assert(io.vecStoreRetire.bits === rdataPtrExt(0)) 740 vec(rdataPtrExt(0).value) := false.B 741 vecAddrvalid(rdataPtrExt(0).value) := false.B 742 allocated(rdataPtrExt(0).value) := false.B 743 } 744 745 val mmioStall = mmio(rdataPtrExt(0).value) 746 val vecStall = vec(rdataPtrExt(0).value) 747 for (i <- 0 until EnsbufferWidth) { 748 val ptr = rdataPtrExt(i).value 749 dataBuffer.io.enq(i).valid := allocated(ptr) && committed(ptr) && !mmioStall && !vecStall 750 // Note that store data/addr should both be valid after store's commit 751 assert(!dataBuffer.io.enq(i).valid || allvalid(ptr)) 752 dataBuffer.io.enq(i).bits.addr := paddrModule.io.rdata(i) 753 dataBuffer.io.enq(i).bits.vaddr := vaddrModule.io.rdata(i) 754 dataBuffer.io.enq(i).bits.data := dataModule.io.rdata(i).data 755 dataBuffer.io.enq(i).bits.mask := dataModule.io.rdata(i).mask 756 dataBuffer.io.enq(i).bits.wline := paddrModule.io.rlineflag(i) 757 dataBuffer.io.enq(i).bits.sqPtr := rdataPtrExt(i) 758 dataBuffer.io.enq(i).bits.prefetch := prefetch(ptr) 759 } 760 761 // Send data stored in sbufferReqBitsReg to sbuffer 762 for (i <- 0 until EnsbufferWidth) { 763 io.sbuffer(i).valid := dataBuffer.io.deq(i).valid 764 dataBuffer.io.deq(i).ready := io.sbuffer(i).ready 765 // Write line request should have all 1 mask 766 assert(!(io.sbuffer(i).valid && io.sbuffer(i).bits.wline && !io.sbuffer(i).bits.mask.andR)) 767 io.sbuffer(i).bits := DontCare 768 io.sbuffer(i).bits.cmd := MemoryOpConstants.M_XWR 769 io.sbuffer(i).bits.addr := dataBuffer.io.deq(i).bits.addr 770 io.sbuffer(i).bits.vaddr := dataBuffer.io.deq(i).bits.vaddr 771 io.sbuffer(i).bits.data := dataBuffer.io.deq(i).bits.data 772 io.sbuffer(i).bits.mask := dataBuffer.io.deq(i).bits.mask 773 io.sbuffer(i).bits.wline := dataBuffer.io.deq(i).bits.wline 774 io.sbuffer(i).bits.prefetch := dataBuffer.io.deq(i).bits.prefetch 775 776 // io.sbuffer(i).fire is RegNexted, as sbuffer data write takes 2 cycles. 777 // Before data write finish, sbuffer is unable to provide store to load 778 // forward data. As an workaround, deqPtrExt and allocated flag update 779 // is delayed so that load can get the right data from store queue. 780 val ptr = dataBuffer.io.deq(i).bits.sqPtr.value 781 when (RegNext(io.sbuffer(i).fire)) { 782 allocated(RegEnable(ptr, io.sbuffer(i).fire)) := false.B 783 XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr) 784 } 785 } 786 (1 until EnsbufferWidth).foreach(i => when(io.sbuffer(i).fire) { assert(io.sbuffer(i - 1).fire) }) 787 if (coreParams.dcacheParametersOpt.isEmpty) { 788 for (i <- 0 until EnsbufferWidth) { 789 val ptr = deqPtrExt(i).value 790 val ram = DifftestMem(64L * 1024 * 1024 * 1024, 8) 791 val wen = allocated(ptr) && committed(ptr) && !mmio(ptr) 792 val waddr = ((paddrModule.io.rdata(i) - "h80000000".U) >> 3).asUInt 793 val wdata = Mux(paddrModule.io.rdata(i)(3), dataModule.io.rdata(i).data(127, 64), dataModule.io.rdata(i).data(63, 0)) 794 val wmask = Mux(paddrModule.io.rdata(i)(3), dataModule.io.rdata(i).mask(15, 8), dataModule.io.rdata(i).mask(7, 0)) 795 when (wen) { 796 ram.write(waddr, wdata.asTypeOf(Vec(8, UInt(8.W))), wmask.asBools) 797 } 798 } 799 } 800 801 // Read vaddr for mem exception 802 io.exceptionAddr.vaddr := vaddrModule.io.rdata(EnsbufferWidth) 803 io.exceptionAddr.gpaddr := gpaddrModule.io.rdata(EnsbufferWidth) 804 // misprediction recovery / exception redirect 805 // invalidate sq term using robIdx 806 val needCancel = Wire(Vec(StoreQueueSize, Bool())) 807 for (i <- 0 until StoreQueueSize) { 808 needCancel(i) := uop(i).robIdx.needFlush(io.brqRedirect) && allocated(i) && !committed(i) 809 when (needCancel(i)) { 810 allocated(i) := false.B 811 } 812 } 813 814 /** 815* update pointers 816**/ 817 val lastEnqCancel = PopCount(RegNext(VecInit(canEnqueue.zip(enqCancel).map(x => x._1 && x._2)))) // 1 cycle after redirect 818 val lastCycleCancelCount = PopCount(RegNext(needCancel)) // 1 cycle after redirect 819 val lastCycleRedirect = RegNext(io.brqRedirect.valid) // 1 cycle after redirect 820 val enqNumber = Mux(!lastCycleRedirect&&io.enq.canAccept && io.enq.lqCanAccept, PopCount(io.enq.req.map(_.valid)), 0.U) // 1 cycle after redirect 821 822 val lastlastCycleRedirect=RegNext(lastCycleRedirect)// 2 cycle after redirect 823 val redirectCancelCount = RegEnable(lastCycleCancelCount + lastEnqCancel, lastCycleRedirect) // 2 cycle after redirect 824 825 when (lastlastCycleRedirect) { 826 // we recover the pointers in 2 cycle after redirect for better timing 827 enqPtrExt := VecInit(enqPtrExt.map(_ - redirectCancelCount)) 828 }.otherwise { 829 // lastCycleRedirect.valid or nornal case 830 // when lastCycleRedirect.valid, enqNumber === 0.U, enqPtrExt will not change 831 enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) 832 } 833 assert(!(lastCycleRedirect && enqNumber =/= 0.U)) 834 835 deqPtrExt := deqPtrExtNext 836 rdataPtrExt := rdataPtrExtNext 837 838 // val dequeueCount = Mux(io.sbuffer(1).fire, 2.U, Mux(io.sbuffer(0).fire || io.mmioStout.fire, 1.U, 0.U)) 839 840 // If redirect at T0, sqCancelCnt is at T2 841 io.sqCancelCnt := redirectCancelCount 842 val ForceWriteUpper = Wire(UInt(log2Up(StoreQueueSize + 1).W)) 843 ForceWriteUpper := Constantin.createRecord("ForceWriteUpper_"+p(XSCoreParamsKey).HartId.toString(), initValue = 60.U) 844 val ForceWriteLower = Wire(UInt(log2Up(StoreQueueSize + 1).W)) 845 ForceWriteLower := Constantin.createRecord("ForceWriteLower_"+p(XSCoreParamsKey).HartId.toString(), initValue = 55.U) 846 847 val valid_cnt = PopCount(allocated) 848 io.force_write := RegNext(Mux(valid_cnt >= ForceWriteUpper, true.B, valid_cnt >= ForceWriteLower && io.force_write), init = false.B) 849 850 // io.sqempty will be used by sbuffer 851 // We delay it for 1 cycle for better timing 852 // When sbuffer need to check if it is empty, the pipeline is blocked, which means delay io.sqempty 853 // for 1 cycle will also promise that sq is empty in that cycle 854 io.sqEmpty := RegNext( 855 enqPtrExt(0).value === deqPtrExt(0).value && 856 enqPtrExt(0).flag === deqPtrExt(0).flag 857 ) 858 // perf counter 859 QueuePerf(StoreQueueSize, validCount, !allowEnqueue) 860 io.sqFull := !allowEnqueue 861 XSPerfAccumulate("mmioCycle", uncacheState =/= s_idle) // lq is busy dealing with uncache req 862 XSPerfAccumulate("mmioCnt", io.uncache.req.fire) 863 XSPerfAccumulate("mmio_wb_success", io.mmioStout.fire) 864 XSPerfAccumulate("mmio_wb_blocked", io.mmioStout.valid && !io.mmioStout.ready) 865 XSPerfAccumulate("validEntryCnt", distanceBetween(enqPtrExt(0), deqPtrExt(0))) 866 XSPerfAccumulate("cmtEntryCnt", distanceBetween(cmtPtrExt(0), deqPtrExt(0))) 867 XSPerfAccumulate("nCmtEntryCnt", distanceBetween(enqPtrExt(0), cmtPtrExt(0))) 868 869 val perfValidCount = distanceBetween(enqPtrExt(0), deqPtrExt(0)) 870 val perfEvents = Seq( 871 ("mmioCycle ", uncacheState =/= s_idle), 872 ("mmioCnt ", io.uncache.req.fire), 873 ("mmio_wb_success", io.mmioStout.fire), 874 ("mmio_wb_blocked", io.mmioStout.valid && !io.mmioStout.ready), 875 ("stq_1_4_valid ", (perfValidCount < (StoreQueueSize.U/4.U))), 876 ("stq_2_4_valid ", (perfValidCount > (StoreQueueSize.U/4.U)) & (perfValidCount <= (StoreQueueSize.U/2.U))), 877 ("stq_3_4_valid ", (perfValidCount > (StoreQueueSize.U/2.U)) & (perfValidCount <= (StoreQueueSize.U*3.U/4.U))), 878 ("stq_4_4_valid ", (perfValidCount > (StoreQueueSize.U*3.U/4.U))), 879 ) 880 generatePerfEvent() 881 882 // debug info 883 XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt(0).flag, deqPtr) 884 885 def PrintFlag(flag: Bool, name: String): Unit = { 886 when(flag) { 887 XSDebug(false, true.B, name) 888 }.otherwise { 889 XSDebug(false, true.B, " ") 890 } 891 } 892 893 for (i <- 0 until StoreQueueSize) { 894 XSDebug(i + ": pc %x va %x pa %x data %x ", 895 uop(i).pc, 896 debug_vaddr(i), 897 debug_paddr(i), 898 debug_data(i) 899 ) 900 PrintFlag(allocated(i), "a") 901 PrintFlag(allocated(i) && addrvalid(i), "a") 902 PrintFlag(allocated(i) && datavalid(i), "d") 903 PrintFlag(allocated(i) && committed(i), "c") 904 PrintFlag(allocated(i) && pending(i), "p") 905 PrintFlag(allocated(i) && mmio(i), "m") 906 XSDebug(false, true.B, "\n") 907 } 908 909} 910