1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan._ 25import xiangshan.backend.fu.FuConfig._ 26import xiangshan.backend.fu.fpu.FPU 27import xiangshan.backend.rob.RobLsqIO 28import xiangshan.cache._ 29import xiangshan.frontend.FtqPtr 30import xiangshan.ExceptionNO._ 31import xiangshan.cache.wpu.ReplayCarry 32import xiangshan.backend.rob.RobPtr 33import xiangshan.backend.Bundles.{MemExuOutput, DynInst} 34 35class StoreMisalignBuffer(implicit p: Parameters) extends XSModule 36 with HasCircularQueuePtrHelper 37{ 38 private val enqPortNum = StorePipelineWidth 39 private val maxSplitNum = 2 40 41 require(maxSplitNum == 2) 42 43 private val SB = "b00".U(2.W) 44 private val SH = "b01".U(2.W) 45 private val SW = "b10".U(2.W) 46 private val SD = "b11".U(2.W) 47 48 // encode of how many bytes to shift or truncate 49 private val BYTE0 = "b000".U(3.W) 50 private val BYTE1 = "b001".U(3.W) 51 private val BYTE2 = "b010".U(3.W) 52 private val BYTE3 = "b011".U(3.W) 53 private val BYTE4 = "b100".U(3.W) 54 private val BYTE5 = "b101".U(3.W) 55 private val BYTE6 = "b110".U(3.W) 56 private val BYTE7 = "b111".U(3.W) 57 58 def getMask(sizeEncode: UInt) = LookupTree(sizeEncode, List( 59 SB -> 0x1.U, 60 SH -> 0x3.U, 61 SW -> 0xf.U, 62 SD -> 0xff.U 63 )) 64 65 def selectOldest[T <: LsPipelineBundle](valid: Seq[Bool], bits: Seq[T]): (Seq[Bool], Seq[T]) = { 66 assert(valid.length == bits.length) 67 if (valid.length == 0 || valid.length == 1) { 68 (valid, bits) 69 } else if (valid.length == 2) { 70 val res = Seq.fill(2)(Wire(ValidIO(chiselTypeOf(bits(0))))) 71 for (i <- res.indices) { 72 res(i).valid := valid(i) 73 res(i).bits := bits(i) 74 } 75 val oldest = Mux(valid(0) && valid(1), 76 Mux(isAfter(bits(0).uop.robIdx, bits(1).uop.robIdx) || 77 (isNotBefore(bits(0).uop.robIdx, bits(1).uop.robIdx) && bits(0).uop.uopIdx > bits(1).uop.uopIdx), res(1), res(0)), 78 Mux(valid(0) && !valid(1), res(0), res(1))) 79 (Seq(oldest.valid), Seq(oldest.bits)) 80 } else { 81 val left = selectOldest(valid.take(valid.length / 2), bits.take(bits.length / 2)) 82 val right = selectOldest(valid.takeRight(valid.length - (valid.length / 2)), bits.takeRight(bits.length - (bits.length / 2))) 83 selectOldest(left._1 ++ right._1, left._2 ++ right._2) 84 } 85 } 86 87 val io = IO(new Bundle() { 88 val redirect = Flipped(Valid(new Redirect)) 89 val req = Vec(enqPortNum, Flipped(Valid(new LsPipelineBundle))) 90 val rob = Flipped(new RobLsqIO) 91 val splitStoreReq = Decoupled(new LsPipelineBundle) 92 val splitStoreResp = Flipped(Valid(new SqWriteBundle)) 93 val writeBack = Decoupled(new MemExuOutput) 94 val overwriteExpBuf = Output(new XSBundle { 95 val valid = Bool() 96 val vaddr = UInt(VAddrBits.W) 97 val gpaddr = UInt(GPAddrBits.W) 98 }) 99 val sqControl = new StoreMaBufToSqControlIO 100 }) 101 102 io.rob.mmio := 0.U.asTypeOf(Vec(LoadPipelineWidth, Bool())) 103 io.rob.uop := 0.U.asTypeOf(Vec(LoadPipelineWidth, new DynInst)) 104 105 val req_valid = RegInit(false.B) 106 val req = Reg(new LsPipelineBundle) 107 108 // enqueue 109 // s1: 110 val s1_req = VecInit(io.req.map(_.bits)) 111 val s1_valid = VecInit(io.req.map(x => x.valid)) 112 113 // s2: delay 1 cycle 114 val s2_req = RegNext(s1_req) 115 val s2_valid = (0 until enqPortNum).map(i => 116 RegNext(s1_valid(i)) && 117 !s2_req(i).uop.robIdx.needFlush(RegNext(io.redirect)) && 118 !s2_req(i).uop.robIdx.needFlush(io.redirect) 119 ) 120 val s2_miss_aligned = s2_req.map(x => 121 x.uop.exceptionVec(storeAddrMisaligned) && !x.uop.exceptionVec(breakPoint) && !TriggerAction.isDmode(x.uop.trigger) 122 ) 123 124 val s2_enqueue = Wire(Vec(enqPortNum, Bool())) 125 for (w <- 0 until enqPortNum) { 126 s2_enqueue(w) := s2_valid(w) && s2_miss_aligned(w) 127 } 128 129 when (req_valid && req.uop.robIdx.needFlush(io.redirect)) { 130 req_valid := s2_enqueue.asUInt.orR 131 } .elsewhen (s2_enqueue.asUInt.orR) { 132 req_valid := req_valid || true.B 133 } 134 135 val reqSel = selectOldest(s2_enqueue, s2_req) 136 137 when (req_valid) { 138 req := Mux( 139 reqSel._1(0) && (isAfter(req.uop.robIdx, reqSel._2(0).uop.robIdx) || (isNotBefore(req.uop.robIdx, reqSel._2(0).uop.robIdx) && req.uop.uopIdx > reqSel._2(0).uop.uopIdx)), 140 reqSel._2(0), 141 req) 142 } .elsewhen (s2_enqueue.asUInt.orR) { 143 req := reqSel._2(0) 144 } 145 146 val robMatch = req_valid && io.rob.pendingst && (io.rob.pendingPtr === req.uop.robIdx) 147 148 // buffer control: 149 // - split miss-aligned store into aligned stores 150 // - send split store to sta and get result from sta 151 // - control sq write to sb 152 // - control sq write this store back 153 val s_idle :: s_split :: s_req :: s_resp :: s_cal :: s_sq_req :: s_wb :: s_wait :: Nil = Enum(8) 154 val bufferState = RegInit(s_idle) 155 val splitStoreReqs = RegInit(VecInit(List.fill(maxSplitNum)(0.U.asTypeOf(new LsPipelineBundle)))) 156 val splitStoreResp = RegInit(VecInit(List.fill(maxSplitNum)(0.U.asTypeOf(new SqWriteBundle)))) 157 val unSentStores = RegInit(0.U(maxSplitNum.W)) 158 val unWriteStores = RegInit(0.U(maxSplitNum.W)) 159 val curPtr = RegInit(0.U(log2Ceil(maxSplitNum).W)) 160 161 // if there is exception or mmio in split store 162 val globalException = RegInit(false.B) 163 val globalMMIO = RegInit(false.B) 164 165 val hasException = ExceptionNO.selectByFu(io.splitStoreResp.bits.uop.exceptionVec, StaCfg).asUInt.orR && !io.splitStoreResp.bits.need_rep 166 val isMMIO = io.splitStoreResp.bits.mmio && !io.splitStoreResp.bits.need_rep 167 168 switch(bufferState) { 169 is (s_idle) { 170 when (robMatch) { 171 bufferState := s_split 172 } 173 } 174 175 is (s_split) { 176 bufferState := s_req 177 } 178 179 is (s_req) { 180 when (io.splitStoreReq.fire) { 181 bufferState := s_resp 182 } 183 } 184 185 is (s_resp) { 186 when (io.splitStoreResp.valid) { 187 val clearOh = UIntToOH(curPtr) 188 when (hasException || isMMIO) { 189 // commit directly when exception ocurs 190 // if any split store reaches mmio space, delegate to software storeAddrMisaligned exception 191 bufferState := s_wb 192 globalException := hasException 193 globalMMIO := isMMIO 194 } .elsewhen(io.splitStoreResp.bits.need_rep || (unSentStores & ~clearOh).orR) { 195 // need replay or still has unsent requests 196 bufferState := s_req 197 } .otherwise { 198 // got result, goto calculate data and control sq 199 bufferState := s_cal 200 } 201 } 202 } 203 204 is (s_cal) { 205 when (io.sqControl.storeInfo.dataReady) { 206 bufferState := s_sq_req 207 curPtr := 0.U 208 } 209 } 210 211 is (s_sq_req) { 212 when (io.sqControl.storeInfo.completeSbTrans) { 213 when (!((unWriteStores & ~UIntToOH(curPtr)).orR)) { 214 bufferState := s_wb 215 } 216 } 217 } 218 219 is (s_wb) { 220 when (io.writeBack.fire) { 221 bufferState := s_wait 222 } 223 } 224 225 is (s_wait) { 226 when (io.rob.scommit =/= 0.U || req.uop.robIdx.needFlush(io.redirect)) { 227 // rob commits the unaligned store or handled the exception, reset all state 228 bufferState := s_idle 229 req_valid := false.B 230 curPtr := 0.U 231 unSentStores := 0.U 232 unWriteStores := 0.U 233 globalException := false.B 234 globalMMIO := false.B 235 } 236 } 237 } 238 239 val highAddress = LookupTree(req.uop.fuOpType(1, 0), List( 240 SB -> 0.U, 241 SH -> 1.U, 242 SW -> 3.U, 243 SD -> 7.U 244 )) + req.vaddr(4, 0) 245 // to see if (vaddr + opSize - 1) and vaddr are in the same 16 bytes region 246 val cross16BytesBoundary = req_valid && (highAddress(4) =/= req.vaddr(4)) 247 val aligned16BytesAddr = (req.vaddr >> 4) << 4// req.vaddr & ~("b1111".U) 248 val aligned16BytesSel = req.vaddr(3, 0) 249 250 // meta of 128 bit store 251 val new128Store = WireInit(0.U.asTypeOf(new LsPipelineBundle)) 252 // meta of split loads 253 val lowAddrStore = WireInit(0.U.asTypeOf(new LsPipelineBundle)) 254 val highAddrStore = WireInit(0.U.asTypeOf(new LsPipelineBundle)) 255 // final lowResult = Cat(`lowResultWidth` of store data, 0.U(make it to fill total length of Vlen)) 256 val lowResultWidth = RegInit(0.U(3.W)) // how many bytes should we take from the store data 257 // final highResult = Zero extend to Vlen(`highResultWidth` of (store data >> lowResultWidth)) 258 val highResultWidth = RegInit(0.U(3.W)) // how many bytes should we take from the store data 259 260 when (bufferState === s_split) { 261 when (!cross16BytesBoundary) { 262 // change this unaligned store into a 128 bits store 263 unWriteStores := 1.U 264 unSentStores := 1.U 265 curPtr := 0.U 266 new128Store.vaddr := aligned16BytesAddr 267 // new128Store.mask := (getMask(req.uop.fuOpType(1, 0)) << aligned16BytesSel).asUInt 268 new128Store.mask := 0xffff.U 269 new128Store.uop := req.uop 270 new128Store.uop.exceptionVec(storeAddrMisaligned) := false.B 271 new128Store.is128bit := true.B 272 splitStoreReqs(0) := new128Store 273 } .otherwise { 274 // split this unaligned store into `maxSplitNum` aligned stores 275 unWriteStores := Fill(maxSplitNum, 1.U(1.W)) 276 unSentStores := Fill(maxSplitNum, 1.U(1.W)) 277 curPtr := 0.U 278 lowAddrStore.uop := req.uop 279 lowAddrStore.uop.exceptionVec(storeAddrMisaligned) := false.B 280 highAddrStore.uop := req.uop 281 highAddrStore.uop.exceptionVec(storeAddrMisaligned) := false.B 282 283 switch (req.uop.fuOpType(1, 0)) { 284 is (SB) { 285 assert(false.B, "lb should not trigger miss align") 286 } 287 288 is (SH) { 289 lowAddrStore.uop.fuOpType := SB 290 lowAddrStore.vaddr := req.vaddr 291 lowAddrStore.mask := 0x1.U << lowAddrStore.vaddr(3, 0) 292 lowResultWidth := BYTE1 293 294 highAddrStore.uop.fuOpType := SB 295 highAddrStore.vaddr := req.vaddr + 1.U 296 highAddrStore.mask := 0x1.U << highAddrStore.vaddr(3, 0) 297 highResultWidth := BYTE1 298 } 299 300 is (SW) { 301 switch (req.vaddr(1, 0)) { 302 is ("b00".U) { 303 assert(false.B, "should not trigger miss align") 304 } 305 306 is ("b01".U) { 307 lowAddrStore.uop.fuOpType := SW 308 lowAddrStore.vaddr := req.vaddr - 1.U 309 lowAddrStore.mask := 0xf.U << lowAddrStore.vaddr(3, 0) 310 lowResultWidth := BYTE3 311 312 highAddrStore.uop.fuOpType := SB 313 highAddrStore.vaddr := req.vaddr + 3.U 314 highAddrStore.mask := 0x1.U << highAddrStore.vaddr(3, 0) 315 highResultWidth := BYTE1 316 } 317 318 is ("b10".U) { 319 lowAddrStore.uop.fuOpType := SH 320 lowAddrStore.vaddr := req.vaddr 321 lowAddrStore.mask := 0x3.U << lowAddrStore.vaddr(3, 0) 322 lowResultWidth := BYTE2 323 324 highAddrStore.uop.fuOpType := SH 325 highAddrStore.vaddr := req.vaddr + 2.U 326 highAddrStore.mask := 0x3.U << highAddrStore.vaddr(3, 0) 327 highResultWidth := BYTE2 328 } 329 330 is ("b11".U) { 331 lowAddrStore.uop.fuOpType := SB 332 lowAddrStore.vaddr := req.vaddr 333 lowAddrStore.mask := 0x1.U << lowAddrStore.vaddr(3, 0) 334 lowResultWidth := BYTE1 335 336 highAddrStore.uop.fuOpType := SW 337 highAddrStore.vaddr := req.vaddr + 1.U 338 highAddrStore.mask := 0xf.U << highAddrStore.vaddr(3, 0) 339 highResultWidth := BYTE3 340 } 341 } 342 } 343 344 is (SD) { 345 switch (req.vaddr(2, 0)) { 346 is ("b000".U) { 347 assert(false.B, "should not trigger miss align") 348 } 349 350 is ("b001".U) { 351 lowAddrStore.uop.fuOpType := SD 352 lowAddrStore.vaddr := req.vaddr - 1.U 353 lowAddrStore.mask := 0xff.U << lowAddrStore.vaddr(3, 0) 354 lowResultWidth := BYTE7 355 356 highAddrStore.uop.fuOpType := SB 357 highAddrStore.vaddr := req.vaddr + 7.U 358 highAddrStore.mask := 0x1.U << highAddrStore.vaddr(3, 0) 359 highResultWidth := BYTE1 360 } 361 362 is ("b010".U) { 363 lowAddrStore.uop.fuOpType := SD 364 lowAddrStore.vaddr := req.vaddr - 2.U 365 lowAddrStore.mask := 0xff.U << lowAddrStore.vaddr(3, 0) 366 lowResultWidth := BYTE6 367 368 highAddrStore.uop.fuOpType := SH 369 highAddrStore.vaddr := req.vaddr + 6.U 370 highAddrStore.mask := 0x3.U << highAddrStore.vaddr(3, 0) 371 highResultWidth := BYTE2 372 } 373 374 is ("b011".U) { 375 lowAddrStore.uop.fuOpType := SD 376 lowAddrStore.vaddr := req.vaddr - 3.U 377 lowAddrStore.mask := 0xff.U << lowAddrStore.vaddr(3, 0) 378 lowResultWidth := BYTE5 379 380 highAddrStore.uop.fuOpType := SW 381 highAddrStore.vaddr := req.vaddr + 5.U 382 highAddrStore.mask := 0xf.U << highAddrStore.vaddr(3, 0) 383 highResultWidth := BYTE3 384 } 385 386 is ("b100".U) { 387 lowAddrStore.uop.fuOpType := SW 388 lowAddrStore.vaddr := req.vaddr 389 lowAddrStore.mask := 0xf.U << lowAddrStore.vaddr(3, 0) 390 lowResultWidth := BYTE4 391 392 highAddrStore.uop.fuOpType := SW 393 highAddrStore.vaddr := req.vaddr + 4.U 394 highAddrStore.mask := 0xf.U << highAddrStore.vaddr(3, 0) 395 highResultWidth := BYTE4 396 } 397 398 is ("b101".U) { 399 lowAddrStore.uop.fuOpType := SD 400 lowAddrStore.vaddr := req.vaddr - 5.U 401 lowAddrStore.mask := 0xff.U << lowAddrStore.vaddr(3, 0) 402 lowResultWidth := BYTE3 403 404 highAddrStore.uop.fuOpType := SD 405 highAddrStore.vaddr := req.vaddr + 3.U 406 highAddrStore.mask := 0xff.U << highAddrStore.vaddr(3, 0) 407 highResultWidth := BYTE5 408 } 409 410 is ("b110".U) { 411 lowAddrStore.uop.fuOpType := SD 412 lowAddrStore.vaddr := req.vaddr - 6.U 413 lowAddrStore.mask := 0xff.U << lowAddrStore.vaddr(3, 0) 414 lowResultWidth := BYTE2 415 416 highAddrStore.uop.fuOpType := SD 417 highAddrStore.vaddr := req.vaddr + 2.U 418 highAddrStore.mask := 0xff.U << highAddrStore.vaddr(3, 0) 419 highResultWidth := BYTE6 420 } 421 422 is ("b111".U) { 423 lowAddrStore.uop.fuOpType := SD 424 lowAddrStore.vaddr := req.vaddr - 7.U 425 lowAddrStore.mask := 0xff.U << lowAddrStore.vaddr(3, 0) 426 lowResultWidth := BYTE1 427 428 highAddrStore.uop.fuOpType := SD 429 highAddrStore.vaddr := req.vaddr + 1.U 430 highAddrStore.mask := 0xff.U << highAddrStore.vaddr(3, 0) 431 highResultWidth := BYTE7 432 } 433 } 434 } 435 } 436 437 splitStoreReqs(0) := lowAddrStore 438 splitStoreReqs(1) := highAddrStore 439 } 440 } 441 442 io.splitStoreReq.valid := req_valid && (bufferState === s_req) 443 io.splitStoreReq.bits := splitStoreReqs(curPtr) 444 445 when (io.splitStoreResp.valid) { 446 splitStoreResp(curPtr) := io.splitStoreResp.bits 447 when (isMMIO) { 448 unWriteStores := 0.U 449 unSentStores := 0.U 450 splitStoreResp(curPtr).uop.exceptionVec := 0.U.asTypeOf(ExceptionVec()) 451 // delegate to software 452 splitStoreResp(curPtr).uop.exceptionVec(storeAddrMisaligned) := true.B 453 } .elsewhen (hasException) { 454 unWriteStores := 0.U 455 unSentStores := 0.U 456 } .elsewhen (!io.splitStoreResp.bits.need_rep) { 457 unSentStores := unSentStores & ~UIntToOH(curPtr) 458 curPtr := curPtr + 1.U 459 } 460 } 461 462 val splitStoreData = RegInit(VecInit(List.fill(maxSplitNum)(0.U.asTypeOf(new XSBundle { 463 val wdata = UInt(VLEN.W) 464 val wmask = UInt((VLEN / 8).W) 465 })))) 466 467 val unalignedStoreData = io.sqControl.storeInfo.data 468 val wmaskLow = Wire(Vec(VLEN / 8, Bool())) 469 val wmaskHigh = Wire(Vec(VLEN / 8, Bool())) 470 (0 until (VLEN / 8)).map { 471 case i => { 472 when (i.U < highResultWidth) { 473 wmaskHigh(i) := true.B 474 } .otherwise { 475 wmaskHigh(i) := false.B 476 } 477 when (i.U < lowResultWidth) { 478 wmaskLow(i) := true.B 479 } .otherwise { 480 wmaskLow(i) := false.B 481 } 482 } 483 } 484 485 when (bufferState === s_cal) { 486 when (!cross16BytesBoundary) { 487 splitStoreData(0).wdata := LookupTree(aligned16BytesSel, List( 488 "b0000".U -> unalignedStoreData, 489 "b0001".U -> Cat(unalignedStoreData, 0.U(( 1 * 8).W)), 490 "b0010".U -> Cat(unalignedStoreData, 0.U(( 2 * 8).W)), 491 "b0011".U -> Cat(unalignedStoreData, 0.U(( 3 * 8).W)), 492 "b0100".U -> Cat(unalignedStoreData, 0.U(( 4 * 8).W)), 493 "b0101".U -> Cat(unalignedStoreData, 0.U(( 5 * 8).W)), 494 "b0110".U -> Cat(unalignedStoreData, 0.U(( 6 * 8).W)), 495 "b0111".U -> Cat(unalignedStoreData, 0.U(( 7 * 8).W)), 496 "b1000".U -> Cat(unalignedStoreData, 0.U(( 8 * 8).W)), 497 "b1001".U -> Cat(unalignedStoreData, 0.U(( 9 * 8).W)), 498 "b1010".U -> Cat(unalignedStoreData, 0.U((10 * 8).W)), 499 "b1011".U -> Cat(unalignedStoreData, 0.U((11 * 8).W)), 500 "b1100".U -> Cat(unalignedStoreData, 0.U((12 * 8).W)), 501 "b1101".U -> Cat(unalignedStoreData, 0.U((13 * 8).W)), 502 "b1110".U -> Cat(unalignedStoreData, 0.U((14 * 8).W)), 503 "b1111".U -> Cat(unalignedStoreData, 0.U((15 * 8).W)) 504 ))(VLEN - 1, 0) 505 splitStoreData(0).wmask := getMask(req.uop.fuOpType(1, 0)) << aligned16BytesSel 506 } .otherwise { 507 // low 16bytes part 508 val catData = LookupTree(lowResultWidth, List( 509 BYTE0 -> unalignedStoreData, 510 BYTE1 -> Cat(unalignedStoreData, 0.U((8 * 15).W)), 511 BYTE2 -> Cat(unalignedStoreData, 0.U((8 * 14).W)), 512 BYTE3 -> Cat(unalignedStoreData, 0.U((8 * 13).W)), 513 BYTE4 -> Cat(unalignedStoreData, 0.U((8 * 12).W)), 514 BYTE5 -> Cat(unalignedStoreData, 0.U((8 * 11).W)), 515 BYTE6 -> Cat(unalignedStoreData, 0.U((8 * 10).W)), 516 BYTE7 -> Cat(unalignedStoreData, 0.U((8 * 9).W)) 517 )) 518 splitStoreData(0).wdata := catData(VLEN - 1, 0) 519 splitStoreData(0).wmask := VecInit(wmaskLow.reverse).asUInt 520 // high 16bytes part 521 val shiftData = LookupTree(lowResultWidth, List( 522 BYTE0 -> unalignedStoreData(VLEN - 1, 0), 523 BYTE1 -> unalignedStoreData(VLEN - 1, 8), 524 BYTE2 -> unalignedStoreData(VLEN - 1, 16), 525 BYTE3 -> unalignedStoreData(VLEN - 1, 24), 526 BYTE4 -> unalignedStoreData(VLEN - 1, 32), 527 BYTE5 -> unalignedStoreData(VLEN - 1, 40), 528 BYTE6 -> unalignedStoreData(VLEN - 1, 48), 529 BYTE7 -> unalignedStoreData(VLEN - 1, 56) 530 )) 531 splitStoreData(1).wdata := LookupTree(highResultWidth, List( 532 BYTE0 -> ZeroExt(shiftData, VLEN), 533 BYTE1 -> ZeroExt(shiftData(7, 0), VLEN), 534 BYTE2 -> ZeroExt(shiftData(15, 0), VLEN), 535 BYTE3 -> ZeroExt(shiftData(23, 0), VLEN), 536 BYTE4 -> ZeroExt(shiftData(31, 0), VLEN), 537 BYTE5 -> ZeroExt(shiftData(39, 0), VLEN), 538 BYTE6 -> ZeroExt(shiftData(47, 0), VLEN), 539 BYTE7 -> ZeroExt(shiftData(55, 0), VLEN) 540 )) 541 splitStoreData(1).wmask := wmaskHigh.asUInt 542 } 543 } 544 545 io.sqControl.control.hasException := req_valid && globalException 546 547 io.sqControl.control.writeSb := bufferState === s_sq_req 548 io.sqControl.control.wdata := splitStoreData(curPtr).wdata 549 io.sqControl.control.wmask := splitStoreData(curPtr).wmask 550 // the paddr and vaddr is not corresponding to the exact addr of 551 io.sqControl.control.paddr := splitStoreResp(curPtr).paddr 552 io.sqControl.control.vaddr := splitStoreResp(curPtr).vaddr 553 io.sqControl.control.last := !((unWriteStores & ~UIntToOH(curPtr)).orR) 554 555 when (bufferState === s_sq_req) { 556 when (io.sqControl.storeInfo.completeSbTrans) { 557 unWriteStores := unWriteStores & ~UIntToOH(curPtr) 558 curPtr := curPtr + 1.U 559 } 560 } 561 562 io.writeBack.valid := req_valid && (bufferState === s_wb) && io.sqControl.storeInfo.dataReady 563 io.writeBack.bits.uop := req.uop 564 io.writeBack.bits.uop.exceptionVec := Mux( 565 globalMMIO || globalException, 566 splitStoreResp(curPtr).uop.exceptionVec, 567 0.U.asTypeOf(ExceptionVec()) // TODO: is this ok? 568 ) 569 io.writeBack.bits.uop.flushPipe := Mux(globalMMIO || globalException, false.B, true.B) 570 io.writeBack.bits.uop.replayInst := false.B 571 io.writeBack.bits.data := unalignedStoreData 572 io.writeBack.bits.debug.isMMIO := globalMMIO 573 io.writeBack.bits.debug.isPerfCnt := false.B 574 io.writeBack.bits.debug.paddr := req.paddr 575 io.writeBack.bits.debug.vaddr := req.vaddr 576 577 io.sqControl.control.removeSq := req_valid && (bufferState === s_wait) && !(globalMMIO || globalException) && (io.rob.scommit =/= 0.U) 578 579 val flush = req_valid && req.uop.robIdx.needFlush(io.redirect) 580 581 when (flush && (bufferState =/= s_idle)) { 582 bufferState := s_idle 583 req_valid := false.B 584 curPtr := 0.U 585 unSentStores := 0.U 586 unWriteStores := 0.U 587 globalException := false.B 588 globalMMIO := false.B 589 } 590 591 // NOTE: spectial case (unaligned store cross page, page fault happens in next page) 592 // if exception happens in the higher page address part, overwrite the storeExceptionBuffer vaddr 593 val overwriteExpBuf = GatedValidRegNext(req_valid && cross16BytesBoundary && globalException && (curPtr === 1.U)) 594 val overwriteAddr = GatedRegNext(splitStoreResp(curPtr).vaddr) 595 val overwriteGpaddr = GatedRegNext(splitStoreResp(curPtr).gpaddr) 596 597 io.overwriteExpBuf.valid := overwriteExpBuf 598 io.overwriteExpBuf.vaddr := overwriteAddr 599 io.overwriteExpBuf.gpaddr := overwriteGpaddr 600 601 XSPerfAccumulate("alloc", RegNext(!req_valid) && req_valid) 602 XSPerfAccumulate("flush", flush) 603 XSPerfAccumulate("flush_idle", flush && (bufferState === s_idle)) 604 XSPerfAccumulate("flush_non_idle", flush && (bufferState =/= s_idle)) 605}