1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19 20import org.chipsalliance.cde.config.Parameters 21import chisel3._ 22import chisel3.util._ 23import utility._ 24import utils._ 25import xiangshan._ 26import xiangshan.backend.Bundles.{DynInst, MemExuInput} 27import xiangshan.backend.rob.RobPtr 28import xiangshan.cache._ 29import xiangshan.backend.fu.FenceToSbuffer 30import xiangshan.cache.wpu.ReplayCarry 31import xiangshan.mem.prefetch.PrefetchReqBundle 32 33object genWmask { 34 def apply(addr: UInt, sizeEncode: UInt): UInt = { 35 (LookupTree(sizeEncode, List( 36 "b00".U -> 0x1.U, //0001 << addr(2:0) 37 "b01".U -> 0x3.U, //0011 38 "b10".U -> 0xf.U, //1111 39 "b11".U -> 0xff.U //11111111 40 )) << addr(2, 0)).asUInt 41 } 42} 43 44object genVWmask { 45 def apply(addr: UInt, sizeEncode: UInt): UInt = { 46 (LookupTree(sizeEncode, List( 47 "b00".U -> 0x1.U, //0001 << addr(2:0) 48 "b01".U -> 0x3.U, //0011 49 "b10".U -> 0xf.U, //1111 50 "b11".U -> 0xff.U //11111111 51 )) << addr(3, 0)).asUInt 52 } 53} 54 55object genWdata { 56 def apply(data: UInt, sizeEncode: UInt): UInt = { 57 LookupTree(sizeEncode, List( 58 "b00".U -> Fill(16, data(7, 0)), 59 "b01".U -> Fill(8, data(15, 0)), 60 "b10".U -> Fill(4, data(31, 0)), 61 "b11".U -> Fill(2, data(63,0)) 62 )) 63 } 64} 65 66object shiftDataToLow { 67 def apply(addr: UInt,data : UInt): UInt = { 68 Mux(addr(3), (data >> 64).asUInt,data) 69 } 70} 71object shiftMaskToLow { 72 def apply(addr: UInt,mask: UInt): UInt = { 73 Mux(addr(3),(mask >> 8).asUInt,mask) 74 } 75} 76 77class LsPipelineBundle(implicit p: Parameters) extends XSBundle 78 with HasDCacheParameters 79 with HasVLSUParameters { 80 val uop = new DynInst 81 val vaddr = UInt(VAddrBits.W) 82 val paddr = UInt(PAddrBits.W) 83 // val func = UInt(6.W) 84 val mask = UInt((VLEN/8).W) 85 val data = UInt((VLEN+1).W) 86 val wlineflag = Bool() // store write the whole cache line 87 88 val miss = Bool() 89 val tlbMiss = Bool() 90 val ptwBack = Bool() 91 val mmio = Bool() 92 val atomic = Bool() 93 val rsIdx = UInt(log2Up(MemIQSizeMax).W) 94 95 val forwardMask = Vec(VLEN/8, Bool()) 96 val forwardData = Vec(VLEN/8, UInt(8.W)) 97 98 // prefetch 99 val isPrefetch = Bool() 100 val isHWPrefetch = Bool() 101 def isSWPrefetch = isPrefetch && !isHWPrefetch 102 103 // vector 104 val isvec = Bool() 105 val isLastElem = Bool() 106 val is128bit = Bool() 107 val uop_unit_stride_fof = Bool() 108 val usSecondInv = Bool() 109 val elemIdx = UInt(elemIdxBits.W) 110 val alignedType = UInt(alignTypeBits.W) 111 // val rob_idx_valid = Vec(2,Bool()) 112 // val inner_idx = Vec(2,UInt(3.W)) 113 // val rob_idx = Vec(2,new RobPtr) 114 val reg_offset = UInt(vOffsetBits.W) 115 // val offset = Vec(2,UInt(4.W)) 116 val vecActive = Bool() // 1: vector active element or scala mem operation, 0: vector not active element 117 val is_first_ele = Bool() 118 // val flowPtr = new VlflowPtr() // VLFlowQueue ptr 119 // val sflowPtr = new VsFlowPtr() // VSFlowQueue ptr 120 121 // For debug usage 122 val isFirstIssue = Bool() 123 val hasROBEntry = Bool() 124 125 // For load replay 126 val isLoadReplay = Bool() 127 val isFastPath = Bool() 128 val isFastReplay = Bool() 129 val replayCarry = new ReplayCarry(nWays) 130 131 // For dcache miss load 132 val mshrid = UInt(log2Up(cfg.nMissEntries).W) 133 val handledByMSHR = Bool() 134 val replacementUpdated = Bool() 135 val missDbUpdated = Bool() 136 137 val forward_tlDchannel = Bool() 138 val dcacheRequireReplay = Bool() 139 val delayedLoadError = Bool() 140 val lateKill = Bool() 141 val feedbacked = Bool() 142 val ldCancel = ValidUndirectioned(UInt(log2Ceil(LoadPipelineWidth).W)) 143 // loadQueueReplay index. 144 val schedIndex = UInt(log2Up(LoadQueueReplaySize).W) 145} 146 147class LdPrefetchTrainBundle(implicit p: Parameters) extends LsPipelineBundle { 148 val meta_prefetch = UInt(L1PfSourceBits.W) 149 val meta_access = Bool() 150 151 def fromLsPipelineBundle(input: LsPipelineBundle, latch: Boolean = false) = { 152 if (latch) vaddr := RegNext(input.vaddr) else vaddr := input.vaddr 153 if (latch) paddr := RegNext(input.paddr) else paddr := input.paddr 154 if (latch) mask := RegNext(input.mask) else mask := input.mask 155 if (latch) data := RegNext(input.data) else data := input.data 156 if (latch) uop := RegNext(input.uop) else uop := input.uop 157 if (latch) wlineflag := RegNext(input.wlineflag) else wlineflag := input.wlineflag 158 if (latch) miss := RegNext(input.miss) else miss := input.miss 159 if (latch) tlbMiss := RegNext(input.tlbMiss) else tlbMiss := input.tlbMiss 160 if (latch) ptwBack := RegNext(input.ptwBack) else ptwBack := input.ptwBack 161 if (latch) mmio := RegNext(input.mmio) else mmio := input.mmio 162 if (latch) rsIdx := RegNext(input.rsIdx) else rsIdx := input.rsIdx 163 if (latch) forwardMask := RegNext(input.forwardMask) else forwardMask := input.forwardMask 164 if (latch) forwardData := RegNext(input.forwardData) else forwardData := input.forwardData 165 if (latch) isPrefetch := RegNext(input.isPrefetch) else isPrefetch := input.isPrefetch 166 if (latch) isHWPrefetch := RegNext(input.isHWPrefetch) else isHWPrefetch := input.isHWPrefetch 167 if (latch) isFirstIssue := RegNext(input.isFirstIssue) else isFirstIssue := input.isFirstIssue 168 if (latch) hasROBEntry := RegNext(input.hasROBEntry) else hasROBEntry := input.hasROBEntry 169 if (latch) dcacheRequireReplay := RegNext(input.dcacheRequireReplay) else dcacheRequireReplay := input.dcacheRequireReplay 170 if (latch) schedIndex := RegNext(input.schedIndex) else schedIndex := input.schedIndex 171 if (latch) isvec := RegNext(input.isvec) else isvec := input.isvec 172 if (latch) isLastElem := RegNext(input.isLastElem) else isLastElem := input.isLastElem 173 if (latch) is128bit := RegNext(input.is128bit) else is128bit := input.is128bit 174 if (latch) vecActive := RegNext(input.vecActive) else vecActive := input.vecActive 175 if (latch) is_first_ele := RegNext(input.is_first_ele) else is_first_ele := input.is_first_ele 176 if (latch) uop_unit_stride_fof := RegNext(input.uop_unit_stride_fof) else uop_unit_stride_fof := input.uop_unit_stride_fof 177 if (latch) usSecondInv := RegNext(input.usSecondInv) else usSecondInv := input.usSecondInv 178 if (latch) reg_offset := RegNext(input.reg_offset) else reg_offset := input.reg_offset 179 if (latch) elemIdx := RegNext(input.elemIdx) else elemIdx := input.elemIdx 180 if (latch) alignedType := RegNext(input.alignedType) else alignedType := input.alignedType 181 // if (latch) flowPtr := RegNext(input.flowPtr) else flowPtr := input.flowPtr 182 // if (latch) sflowPtr := RegNext(input.sflowPtr) else sflowPtr := input.sflowPtr 183 184 meta_prefetch := DontCare 185 meta_access := DontCare 186 forward_tlDchannel := DontCare 187 mshrid := DontCare 188 replayCarry := DontCare 189 atomic := DontCare 190 isLoadReplay := DontCare 191 isFastPath := DontCare 192 isFastReplay := DontCare 193 handledByMSHR := DontCare 194 replacementUpdated := DontCare 195 missDbUpdated := DontCare 196 delayedLoadError := DontCare 197 lateKill := DontCare 198 feedbacked := DontCare 199 ldCancel := DontCare 200 } 201 202 def asPrefetchReqBundle(): PrefetchReqBundle = { 203 val res = Wire(new PrefetchReqBundle) 204 res.vaddr := this.vaddr 205 res.paddr := this.paddr 206 res.pc := this.uop.pc 207 208 res 209 } 210} 211 212class StPrefetchTrainBundle(implicit p: Parameters) extends LdPrefetchTrainBundle {} 213 214class LqWriteBundle(implicit p: Parameters) extends LsPipelineBundle { 215 // load inst replay informations 216 val rep_info = new LoadToLsqReplayIO 217 // queue entry data, except flag bits, will be updated if writeQueue is true, 218 // valid bit in LqWriteBundle will be ignored 219 val data_wen_dup = Vec(6, Bool()) // dirty reg dup 220 221 222 def fromLsPipelineBundle(input: LsPipelineBundle, latch: Boolean = false) = { 223 if(latch) vaddr := RegNext(input.vaddr) else vaddr := input.vaddr 224 if(latch) paddr := RegNext(input.paddr) else paddr := input.paddr 225 if(latch) mask := RegNext(input.mask) else mask := input.mask 226 if(latch) data := RegNext(input.data) else data := input.data 227 if(latch) uop := RegNext(input.uop) else uop := input.uop 228 if(latch) wlineflag := RegNext(input.wlineflag) else wlineflag := input.wlineflag 229 if(latch) miss := RegNext(input.miss) else miss := input.miss 230 if(latch) tlbMiss := RegNext(input.tlbMiss) else tlbMiss := input.tlbMiss 231 if(latch) ptwBack := RegNext(input.ptwBack) else ptwBack := input.ptwBack 232 if(latch) mmio := RegNext(input.mmio) else mmio := input.mmio 233 if(latch) atomic := RegNext(input.atomic) else atomic := input.atomic 234 if(latch) rsIdx := RegNext(input.rsIdx) else rsIdx := input.rsIdx 235 if(latch) forwardMask := RegNext(input.forwardMask) else forwardMask := input.forwardMask 236 if(latch) forwardData := RegNext(input.forwardData) else forwardData := input.forwardData 237 if(latch) isPrefetch := RegNext(input.isPrefetch) else isPrefetch := input.isPrefetch 238 if(latch) isHWPrefetch := RegNext(input.isHWPrefetch) else isHWPrefetch := input.isHWPrefetch 239 if(latch) isFirstIssue := RegNext(input.isFirstIssue) else isFirstIssue := input.isFirstIssue 240 if(latch) hasROBEntry := RegNext(input.hasROBEntry) else hasROBEntry := input.hasROBEntry 241 if(latch) isLoadReplay := RegNext(input.isLoadReplay) else isLoadReplay := input.isLoadReplay 242 if(latch) isFastPath := RegNext(input.isFastPath) else isFastPath := input.isFastPath 243 if(latch) isFastReplay := RegNext(input.isFastReplay) else isFastReplay := input.isFastReplay 244 if(latch) mshrid := RegNext(input.mshrid) else mshrid := input.mshrid 245 if(latch) forward_tlDchannel := RegNext(input.forward_tlDchannel) else forward_tlDchannel := input.forward_tlDchannel 246 if(latch) replayCarry := RegNext(input.replayCarry) else replayCarry := input.replayCarry 247 if(latch) dcacheRequireReplay := RegNext(input.dcacheRequireReplay) else dcacheRequireReplay := input.dcacheRequireReplay 248 if(latch) schedIndex := RegNext(input.schedIndex) else schedIndex := input.schedIndex 249 if(latch) handledByMSHR := RegNext(input.handledByMSHR) else handledByMSHR := input.handledByMSHR 250 if(latch) replacementUpdated := RegNext(input.replacementUpdated) else replacementUpdated := input.replacementUpdated 251 if(latch) missDbUpdated := RegNext(input.missDbUpdated) else missDbUpdated := input.missDbUpdated 252 if(latch) delayedLoadError := RegNext(input.delayedLoadError) else delayedLoadError := input.delayedLoadError 253 if(latch) lateKill := RegNext(input.lateKill) else lateKill := input.lateKill 254 if(latch) feedbacked := RegNext(input.feedbacked) else feedbacked := input.feedbacked 255 if(latch) isvec := RegNext(input.isvec) else isvec := input.isvec 256 if(latch) is128bit := RegNext(input.is128bit) else is128bit := input.is128bit 257 if(latch) vecActive := RegNext(input.vecActive) else vecActive := input.vecActive 258 if(latch) uop_unit_stride_fof := RegNext(input.uop_unit_stride_fof) else uop_unit_stride_fof := input.uop_unit_stride_fof 259 if(latch) reg_offset := RegNext(input.reg_offset) else reg_offset := input.reg_offset 260 261 rep_info := DontCare 262 data_wen_dup := DontCare 263 } 264} 265 266class LoadForwardQueryIO(implicit p: Parameters) extends XSBundle { 267 val vaddr = Output(UInt(VAddrBits.W)) 268 val paddr = Output(UInt(PAddrBits.W)) 269 val mask = Output(UInt((VLEN/8).W)) 270 val uop = Output(new DynInst) // for replay 271 val pc = Output(UInt(VAddrBits.W)) //for debug 272 val valid = Output(Bool()) 273 274 val forwardMaskFast = Input(Vec((VLEN/8), Bool())) // resp to load_s1 275 val forwardMask = Input(Vec((VLEN/8), Bool())) // resp to load_s2 276 val forwardData = Input(Vec((VLEN/8), UInt(8.W))) // resp to load_s2 277 278 // val lqIdx = Output(UInt(LoadQueueIdxWidth.W)) 279 val sqIdx = Output(new SqPtr) 280 281 // dataInvalid suggests store to load forward found forward should happen, 282 // but data is not available for now. If dataInvalid, load inst should 283 // be replayed from RS. Feedback type should be RSFeedbackType.dataInvalid 284 val dataInvalid = Input(Bool()) // Addr match, but data is not valid for now 285 286 // matchInvalid suggests in store to load forward logic, paddr cam result does 287 // to equal to vaddr cam result. If matchInvalid, a microarchitectural exception 288 // should be raised to flush SQ and committed sbuffer. 289 val matchInvalid = Input(Bool()) // resp to load_s2 290 291 // addrInvalid suggests store to load forward found forward should happen, 292 // but address (SSID) is not available for now. If addrInvalid, load inst should 293 // be replayed from RS. Feedback type should be RSFeedbackType.addrInvalid 294 val addrInvalid = Input(Bool()) 295} 296 297// LoadForwardQueryIO used in load pipeline 298// 299// Difference between PipeLoadForwardQueryIO and LoadForwardQueryIO: 300// PipeIO use predecoded sqIdxMask for better forward timing 301class PipeLoadForwardQueryIO(implicit p: Parameters) extends LoadForwardQueryIO { 302 // val sqIdx = Output(new SqPtr) // for debug, should not be used in pipeline for timing reasons 303 // sqIdxMask is calcuated in earlier stage for better timing 304 val sqIdxMask = Output(UInt(StoreQueueSize.W)) 305 306 // dataInvalid: addr match, but data is not valid for now 307 val dataInvalidFast = Input(Bool()) // resp to load_s1 308 // val dataInvalid = Input(Bool()) // resp to load_s2 309 val dataInvalidSqIdx = Input(new SqPtr) // resp to load_s2, sqIdx 310 val addrInvalidSqIdx = Input(new SqPtr) // resp to load_s2, sqIdx 311} 312 313// Query load queue for ld-ld violation 314// 315// Req should be send in load_s1 316// Resp will be generated 1 cycle later 317// 318// Note that query req may be !ready, as dcache is releasing a block 319// If it happens, a replay from rs is needed. 320class LoadNukeQueryReq(implicit p: Parameters) extends XSBundle { // provide lqIdx 321 val uop = new DynInst 322 // mask: load's data mask. 323 val mask = UInt((VLEN/8).W) 324 325 // paddr: load's paddr. 326 val paddr = UInt(PAddrBits.W) 327 // dataInvalid: load data is invalid. 328 val data_valid = Bool() 329} 330 331class LoadNukeQueryResp(implicit p: Parameters) extends XSBundle { 332 // rep_frm_fetch: ld-ld violation check success, replay from fetch. 333 val rep_frm_fetch = Bool() 334} 335 336class LoadNukeQueryIO(implicit p: Parameters) extends XSBundle { 337 val req = Decoupled(new LoadNukeQueryReq) 338 val resp = Flipped(Valid(new LoadNukeQueryResp)) 339 val revoke = Output(Bool()) 340} 341 342class StoreNukeQueryIO(implicit p: Parameters) extends XSBundle { 343 // robIdx: Requestor's (a store instruction) rob index for match logic. 344 val robIdx = new RobPtr 345 346 // paddr: requestor's (a store instruction) physical address for match logic. 347 val paddr = UInt(PAddrBits.W) 348 349 // mask: requestor's (a store instruction) data width mask for match logic. 350 val mask = UInt((VLEN/8).W) 351} 352 353// Store byte valid mask write bundle 354// 355// Store byte valid mask write to SQ takes 2 cycles 356class StoreMaskBundle(implicit p: Parameters) extends XSBundle { 357 val sqIdx = new SqPtr 358 val mask = UInt((VLEN/8).W) 359} 360 361class LoadDataFromDcacheBundle(implicit p: Parameters) extends DCacheBundle { 362 // old dcache: optimize data sram read fanout 363 // val bankedDcacheData = Vec(DCacheBanks, UInt(64.W)) 364 // val bank_oh = UInt(DCacheBanks.W) 365 366 // new dcache 367 val respDcacheData = UInt(VLEN.W) 368 val forwardMask = Vec(VLEN/8, Bool()) 369 val forwardData = Vec(VLEN/8, UInt(8.W)) 370 val uop = new DynInst // for data selection, only fwen and fuOpType are used 371 val addrOffset = UInt(4.W) // for data selection 372 373 // forward tilelink D channel 374 val forward_D = Bool() 375 val forwardData_D = Vec(VLEN/8, UInt(8.W)) 376 377 // forward mshr data 378 val forward_mshr = Bool() 379 val forwardData_mshr = Vec(VLEN/8, UInt(8.W)) 380 381 val forward_result_valid = Bool() 382 383 def dcacheData(): UInt = { 384 // old dcache 385 // val dcache_data = Mux1H(bank_oh, bankedDcacheData) 386 // new dcache 387 val dcache_data = respDcacheData 388 val use_D = forward_D && forward_result_valid 389 val use_mshr = forward_mshr && forward_result_valid 390 Mux(use_D, forwardData_D.asUInt, Mux(use_mshr, forwardData_mshr.asUInt, dcache_data)) 391 } 392 393 def mergedData(): UInt = { 394 val rdataVec = VecInit((0 until VLEN / 8).map(j => 395 Mux(forwardMask(j), forwardData(j), dcacheData()(8*(j+1)-1, 8*j)) 396 )) 397 rdataVec.asUInt 398 } 399} 400 401// Load writeback data from load queue (refill) 402class LoadDataFromLQBundle(implicit p: Parameters) extends XSBundle { 403 val lqData = UInt(64.W) // load queue has merged data 404 val uop = new DynInst // for data selection, only fwen and fuOpType are used 405 val addrOffset = UInt(3.W) // for data selection 406 407 def mergedData(): UInt = { 408 lqData 409 } 410} 411 412// Bundle for load / store wait waking up 413class MemWaitUpdateReq(implicit p: Parameters) extends XSBundle { 414 val robIdx = Vec(backendParams.StaExuCnt, ValidIO(new RobPtr)) 415 val sqIdx = Vec(backendParams.StdCnt, ValidIO(new SqPtr)) 416} 417 418object AddPipelineReg { 419 class PipelineRegModule[T <: Data](gen: T) extends Module { 420 val io = IO(new Bundle() { 421 val in = Flipped(DecoupledIO(gen.cloneType)) 422 val out = DecoupledIO(gen.cloneType) 423 val isFlush = Input(Bool()) 424 }) 425 426 val valid = RegInit(false.B) 427 valid.suggestName("pipeline_reg_valid") 428 when (io.out.fire) { valid := false.B } 429 when (io.in.fire) { valid := true.B } 430 when (io.isFlush) { valid := false.B } 431 432 io.in.ready := !valid || io.out.ready 433 io.out.bits := RegEnable(io.in.bits, io.in.fire) 434 io.out.valid := valid //&& !isFlush 435 } 436 437 def apply[T <: Data] 438 (left: DecoupledIO[T], right: DecoupledIO[T], isFlush: Bool, 439 moduleName: Option[String] = None 440 ){ 441 val pipelineReg = Module(new PipelineRegModule[T](left.bits.cloneType)) 442 if(moduleName.nonEmpty) pipelineReg.suggestName(moduleName.get) 443 pipelineReg.io.in <> left 444 right <> pipelineReg.io.out 445 pipelineReg.io.isFlush := isFlush 446 } 447} 448