1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19 20import org.chipsalliance.cde.config.Parameters 21import chisel3._ 22import chisel3.util._ 23import utility._ 24import utils._ 25import xiangshan._ 26import xiangshan.backend.Bundles.{DynInst, MemExuInput} 27import xiangshan.backend.rob.RobPtr 28import xiangshan.cache._ 29import xiangshan.backend.fu.FenceToSbuffer 30import xiangshan.cache.wpu.ReplayCarry 31import xiangshan.mem.prefetch.PrefetchReqBundle 32import math._ 33 34object genWmask { 35 def apply(addr: UInt, sizeEncode: UInt): UInt = { 36 (LookupTree(sizeEncode, List( 37 "b00".U -> 0x1.U, //0001 << addr(2:0) 38 "b01".U -> 0x3.U, //0011 39 "b10".U -> 0xf.U, //1111 40 "b11".U -> 0xff.U //11111111 41 )) << addr(2, 0)).asUInt 42 } 43} 44 45object genVWmask { 46 def apply(addr: UInt, sizeEncode: UInt): UInt = { 47 (LookupTree(sizeEncode, List( 48 "b00".U -> 0x1.U, //0001 << addr(2:0) 49 "b01".U -> 0x3.U, //0011 50 "b10".U -> 0xf.U, //1111 51 "b11".U -> 0xff.U //11111111 52 )) << addr(3, 0)).asUInt 53 } 54} 55 56object genWdata { 57 def apply(data: UInt, sizeEncode: UInt): UInt = { 58 LookupTree(sizeEncode, List( 59 "b00".U -> Fill(16, data(7, 0)), 60 "b01".U -> Fill(8, data(15, 0)), 61 "b10".U -> Fill(4, data(31, 0)), 62 "b11".U -> Fill(2, data(63,0)) 63 )) 64 } 65} 66 67object shiftDataToLow { 68 def apply(addr: UInt,data : UInt): UInt = { 69 Mux(addr(3), (data >> 64).asUInt,data) 70 } 71} 72object shiftMaskToLow { 73 def apply(addr: UInt,mask: UInt): UInt = { 74 Mux(addr(3),(mask >> 8).asUInt,mask) 75 } 76} 77 78class LsPipelineBundle(implicit p: Parameters) extends XSBundle 79 with HasDCacheParameters 80 with HasVLSUParameters { 81 val uop = new DynInst 82 val vaddr = UInt(VAddrBits.W) 83 val paddr = UInt(PAddrBits.W) 84 val gpaddr = UInt(GPAddrBits.W) 85 // val func = UInt(6.W) 86 val mask = UInt((VLEN/8).W) 87 val data = UInt((VLEN+1).W) 88 val wlineflag = Bool() // store write the whole cache line 89 90 val miss = Bool() 91 val tlbMiss = Bool() 92 val ptwBack = Bool() 93 val af = Bool() 94 val mmio = Bool() 95 val atomic = Bool() 96 val rsIdx = UInt(log2Up(MemIQSizeMax).W) 97 98 val forwardMask = Vec(VLEN/8, Bool()) 99 val forwardData = Vec(VLEN/8, UInt(8.W)) 100 101 // prefetch 102 val isPrefetch = Bool() 103 val isHWPrefetch = Bool() 104 def isSWPrefetch = isPrefetch && !isHWPrefetch 105 106 // vector 107 val isvec = Bool() 108 val isLastElem = Bool() 109 val is128bit = Bool() 110 val uop_unit_stride_fof = Bool() 111 val usSecondInv = Bool() 112 val elemIdx = UInt(elemIdxBits.W) 113 val alignedType = UInt(alignTypeBits.W) 114 val mbIndex = UInt(max(vlmBindexBits, vsmBindexBits).W) 115 // val rob_idx_valid = Vec(2,Bool()) 116 // val inner_idx = Vec(2,UInt(3.W)) 117 // val rob_idx = Vec(2,new RobPtr) 118 val reg_offset = UInt(vOffsetBits.W) 119 val elemIdxInsideVd = UInt(elemIdxBits.W) 120 // val offset = Vec(2,UInt(4.W)) 121 val vecActive = Bool() // 1: vector active element or scala mem operation, 0: vector not active element 122 val is_first_ele = Bool() 123 // val flowPtr = new VlflowPtr() // VLFlowQueue ptr 124 // val sflowPtr = new VsFlowPtr() // VSFlowQueue ptr 125 126 // For debug usage 127 val isFirstIssue = Bool() 128 val hasROBEntry = Bool() 129 130 // For load replay 131 val isLoadReplay = Bool() 132 val isFastPath = Bool() 133 val isFastReplay = Bool() 134 val replayCarry = new ReplayCarry(nWays) 135 136 // For dcache miss load 137 val mshrid = UInt(log2Up(cfg.nMissEntries).W) 138 val handledByMSHR = Bool() 139 val replacementUpdated = Bool() 140 val missDbUpdated = Bool() 141 142 val forward_tlDchannel = Bool() 143 val dcacheRequireReplay = Bool() 144 val delayedLoadError = Bool() 145 val lateKill = Bool() 146 val feedbacked = Bool() 147 val ldCancel = ValidUndirectioned(UInt(log2Ceil(LoadPipelineWidth).W)) 148 // loadQueueReplay index. 149 val schedIndex = UInt(log2Up(LoadQueueReplaySize).W) 150} 151 152class LdPrefetchTrainBundle(implicit p: Parameters) extends LsPipelineBundle { 153 val meta_prefetch = UInt(L1PfSourceBits.W) 154 val meta_access = Bool() 155 156 def fromLsPipelineBundle(input: LsPipelineBundle, latch: Boolean = false, enable: Bool = true.B) = { 157 if (latch) vaddr := RegEnable(input.vaddr, enable) else vaddr := input.vaddr 158 if (latch) paddr := RegEnable(input.paddr, enable) else paddr := input.paddr 159 if (latch) gpaddr := RegEnable(input.gpaddr, enable) else gpaddr := input.gpaddr 160 if (latch) mask := RegEnable(input.mask, enable) else mask := input.mask 161 if (latch) data := RegEnable(input.data, enable) else data := input.data 162 if (latch) uop := RegEnable(input.uop, enable) else uop := input.uop 163 if (latch) wlineflag := RegEnable(input.wlineflag, enable) else wlineflag := input.wlineflag 164 if (latch) miss := RegEnable(input.miss, enable) else miss := input.miss 165 if (latch) tlbMiss := RegEnable(input.tlbMiss, enable) else tlbMiss := input.tlbMiss 166 if (latch) ptwBack := RegEnable(input.ptwBack, enable) else ptwBack := input.ptwBack 167 if (latch) af := RegEnable(input.af, enable) else af := input.af 168 if (latch) mmio := RegEnable(input.mmio, enable) else mmio := input.mmio 169 if (latch) rsIdx := RegEnable(input.rsIdx, enable) else rsIdx := input.rsIdx 170 if (latch) forwardMask := RegEnable(input.forwardMask, enable) else forwardMask := input.forwardMask 171 if (latch) forwardData := RegEnable(input.forwardData, enable) else forwardData := input.forwardData 172 if (latch) isPrefetch := RegEnable(input.isPrefetch, enable) else isPrefetch := input.isPrefetch 173 if (latch) isHWPrefetch := RegEnable(input.isHWPrefetch, enable) else isHWPrefetch := input.isHWPrefetch 174 if (latch) isFirstIssue := RegEnable(input.isFirstIssue, enable) else isFirstIssue := input.isFirstIssue 175 if (latch) hasROBEntry := RegEnable(input.hasROBEntry, enable) else hasROBEntry := input.hasROBEntry 176 if (latch) dcacheRequireReplay := RegEnable(input.dcacheRequireReplay, enable) else dcacheRequireReplay := input.dcacheRequireReplay 177 if (latch) schedIndex := RegEnable(input.schedIndex, enable) else schedIndex := input.schedIndex 178 if (latch) isvec := RegEnable(input.isvec, enable) else isvec := input.isvec 179 if (latch) isLastElem := RegEnable(input.isLastElem, enable) else isLastElem := input.isLastElem 180 if (latch) is128bit := RegEnable(input.is128bit, enable) else is128bit := input.is128bit 181 if (latch) vecActive := RegEnable(input.vecActive, enable) else vecActive := input.vecActive 182 if (latch) is_first_ele := RegEnable(input.is_first_ele, enable) else is_first_ele := input.is_first_ele 183 if (latch) uop_unit_stride_fof := RegEnable(input.uop_unit_stride_fof, enable) else uop_unit_stride_fof := input.uop_unit_stride_fof 184 if (latch) usSecondInv := RegEnable(input.usSecondInv, enable) else usSecondInv := input.usSecondInv 185 if (latch) reg_offset := RegEnable(input.reg_offset, enable) else reg_offset := input.reg_offset 186 if (latch) elemIdx := RegEnable(input.elemIdx, enable) else elemIdx := input.elemIdx 187 if (latch) alignedType := RegEnable(input.alignedType, enable) else alignedType := input.alignedType 188 if (latch) mbIndex := RegEnable(input.mbIndex, enable) else mbIndex := input.mbIndex 189 if (latch) elemIdxInsideVd := RegEnable(input.elemIdxInsideVd, enable) else elemIdxInsideVd := input.elemIdxInsideVd 190 // if (latch) flowPtr := RegEnable(input.flowPtr, enable) else flowPtr := input.flowPtr 191 // if (latch) sflowPtr := RegEnable(input.sflowPtr, enable) else sflowPtr := input.sflowPtr 192 193 meta_prefetch := DontCare 194 meta_access := DontCare 195 forward_tlDchannel := DontCare 196 mshrid := DontCare 197 replayCarry := DontCare 198 atomic := DontCare 199 isLoadReplay := DontCare 200 isFastPath := DontCare 201 isFastReplay := DontCare 202 handledByMSHR := DontCare 203 replacementUpdated := DontCare 204 missDbUpdated := DontCare 205 delayedLoadError := DontCare 206 lateKill := DontCare 207 feedbacked := DontCare 208 ldCancel := DontCare 209 } 210 211 def asPrefetchReqBundle(): PrefetchReqBundle = { 212 val res = Wire(new PrefetchReqBundle) 213 res.vaddr := this.vaddr 214 res.paddr := this.paddr 215 res.pc := this.uop.pc 216 res.miss := this.miss 217 218 res 219 } 220} 221 222class StPrefetchTrainBundle(implicit p: Parameters) extends LdPrefetchTrainBundle {} 223 224class LqWriteBundle(implicit p: Parameters) extends LsPipelineBundle { 225 // load inst replay informations 226 val rep_info = new LoadToLsqReplayIO 227 // queue entry data, except flag bits, will be updated if writeQueue is true, 228 // valid bit in LqWriteBundle will be ignored 229 val data_wen_dup = Vec(6, Bool()) // dirty reg dup 230 231 232 def fromLsPipelineBundle(input: LsPipelineBundle, latch: Boolean = false, enable: Bool = true.B) = { 233 if(latch) vaddr := RegEnable(input.vaddr, enable) else vaddr := input.vaddr 234 if(latch) paddr := RegEnable(input.paddr, enable) else paddr := input.paddr 235 if(latch) gpaddr := RegEnable(input.gpaddr, enable) else gpaddr := input.gpaddr 236 if(latch) mask := RegEnable(input.mask, enable) else mask := input.mask 237 if(latch) data := RegEnable(input.data, enable) else data := input.data 238 if(latch) uop := RegEnable(input.uop, enable) else uop := input.uop 239 if(latch) wlineflag := RegEnable(input.wlineflag, enable) else wlineflag := input.wlineflag 240 if(latch) miss := RegEnable(input.miss, enable) else miss := input.miss 241 if(latch) tlbMiss := RegEnable(input.tlbMiss, enable) else tlbMiss := input.tlbMiss 242 if(latch) ptwBack := RegEnable(input.ptwBack, enable) else ptwBack := input.ptwBack 243 if(latch) mmio := RegEnable(input.mmio, enable) else mmio := input.mmio 244 if(latch) atomic := RegEnable(input.atomic, enable) else atomic := input.atomic 245 if(latch) rsIdx := RegEnable(input.rsIdx, enable) else rsIdx := input.rsIdx 246 if(latch) forwardMask := RegEnable(input.forwardMask, enable) else forwardMask := input.forwardMask 247 if(latch) forwardData := RegEnable(input.forwardData, enable) else forwardData := input.forwardData 248 if(latch) isPrefetch := RegEnable(input.isPrefetch, enable) else isPrefetch := input.isPrefetch 249 if(latch) isHWPrefetch := RegEnable(input.isHWPrefetch, enable) else isHWPrefetch := input.isHWPrefetch 250 if(latch) isFirstIssue := RegEnable(input.isFirstIssue, enable) else isFirstIssue := input.isFirstIssue 251 if(latch) hasROBEntry := RegEnable(input.hasROBEntry, enable) else hasROBEntry := input.hasROBEntry 252 if(latch) isLoadReplay := RegEnable(input.isLoadReplay, enable) else isLoadReplay := input.isLoadReplay 253 if(latch) isFastPath := RegEnable(input.isFastPath, enable) else isFastPath := input.isFastPath 254 if(latch) isFastReplay := RegEnable(input.isFastReplay, enable) else isFastReplay := input.isFastReplay 255 if(latch) mshrid := RegEnable(input.mshrid, enable) else mshrid := input.mshrid 256 if(latch) forward_tlDchannel := RegEnable(input.forward_tlDchannel, enable) else forward_tlDchannel := input.forward_tlDchannel 257 if(latch) replayCarry := RegEnable(input.replayCarry, enable) else replayCarry := input.replayCarry 258 if(latch) dcacheRequireReplay := RegEnable(input.dcacheRequireReplay, enable) else dcacheRequireReplay := input.dcacheRequireReplay 259 if(latch) schedIndex := RegEnable(input.schedIndex, enable) else schedIndex := input.schedIndex 260 if(latch) handledByMSHR := RegEnable(input.handledByMSHR, enable) else handledByMSHR := input.handledByMSHR 261 if(latch) replacementUpdated := RegEnable(input.replacementUpdated, enable) else replacementUpdated := input.replacementUpdated 262 if(latch) missDbUpdated := RegEnable(input.missDbUpdated, enable) else missDbUpdated := input.missDbUpdated 263 if(latch) delayedLoadError := RegEnable(input.delayedLoadError, enable) else delayedLoadError := input.delayedLoadError 264 if(latch) lateKill := RegEnable(input.lateKill, enable) else lateKill := input.lateKill 265 if(latch) feedbacked := RegEnable(input.feedbacked, enable) else feedbacked := input.feedbacked 266 if(latch) isvec := RegEnable(input.isvec, enable) else isvec := input.isvec 267 if(latch) is128bit := RegEnable(input.is128bit, enable) else is128bit := input.is128bit 268 if(latch) vecActive := RegEnable(input.vecActive, enable) else vecActive := input.vecActive 269 if(latch) uop_unit_stride_fof := RegEnable(input.uop_unit_stride_fof, enable) else uop_unit_stride_fof := input.uop_unit_stride_fof 270 if(latch) reg_offset := RegEnable(input.reg_offset, enable) else reg_offset := input.reg_offset 271 if(latch) mbIndex := RegEnable(input.mbIndex, enable) else mbIndex := input.mbIndex 272 if(latch) elemIdxInsideVd := RegEnable(input.elemIdxInsideVd, enable) else elemIdxInsideVd := input.elemIdxInsideVd 273 274 rep_info := DontCare 275 data_wen_dup := DontCare 276 } 277} 278 279class LoadForwardQueryIO(implicit p: Parameters) extends XSBundle { 280 val vaddr = Output(UInt(VAddrBits.W)) 281 val paddr = Output(UInt(PAddrBits.W)) 282 val mask = Output(UInt((VLEN/8).W)) 283 val uop = Output(new DynInst) // for replay 284 val pc = Output(UInt(VAddrBits.W)) //for debug 285 val valid = Output(Bool()) 286 287 val forwardMaskFast = Input(Vec((VLEN/8), Bool())) // resp to load_s1 288 val forwardMask = Input(Vec((VLEN/8), Bool())) // resp to load_s2 289 val forwardData = Input(Vec((VLEN/8), UInt(8.W))) // resp to load_s2 290 291 // val lqIdx = Output(UInt(LoadQueueIdxWidth.W)) 292 val sqIdx = Output(new SqPtr) 293 294 // dataInvalid suggests store to load forward found forward should happen, 295 // but data is not available for now. If dataInvalid, load inst should 296 // be replayed from RS. Feedback type should be RSFeedbackType.dataInvalid 297 val dataInvalid = Input(Bool()) // Addr match, but data is not valid for now 298 299 // matchInvalid suggests in store to load forward logic, paddr cam result does 300 // to equal to vaddr cam result. If matchInvalid, a microarchitectural exception 301 // should be raised to flush SQ and committed sbuffer. 302 val matchInvalid = Input(Bool()) // resp to load_s2 303 304 // addrInvalid suggests store to load forward found forward should happen, 305 // but address (SSID) is not available for now. If addrInvalid, load inst should 306 // be replayed from RS. Feedback type should be RSFeedbackType.addrInvalid 307 val addrInvalid = Input(Bool()) 308} 309 310// LoadForwardQueryIO used in load pipeline 311// 312// Difference between PipeLoadForwardQueryIO and LoadForwardQueryIO: 313// PipeIO use predecoded sqIdxMask for better forward timing 314class PipeLoadForwardQueryIO(implicit p: Parameters) extends LoadForwardQueryIO { 315 // val sqIdx = Output(new SqPtr) // for debug, should not be used in pipeline for timing reasons 316 // sqIdxMask is calcuated in earlier stage for better timing 317 val sqIdxMask = Output(UInt(StoreQueueSize.W)) 318 319 // dataInvalid: addr match, but data is not valid for now 320 val dataInvalidFast = Input(Bool()) // resp to load_s1 321 // val dataInvalid = Input(Bool()) // resp to load_s2 322 val dataInvalidSqIdx = Input(new SqPtr) // resp to load_s2, sqIdx 323 val addrInvalidSqIdx = Input(new SqPtr) // resp to load_s2, sqIdx 324} 325 326// Query load queue for ld-ld violation 327// 328// Req should be send in load_s1 329// Resp will be generated 1 cycle later 330// 331// Note that query req may be !ready, as dcache is releasing a block 332// If it happens, a replay from rs is needed. 333class LoadNukeQueryReq(implicit p: Parameters) extends XSBundle { // provide lqIdx 334 val uop = new DynInst 335 // mask: load's data mask. 336 val mask = UInt((VLEN/8).W) 337 338 // paddr: load's paddr. 339 val paddr = UInt(PAddrBits.W) 340 // dataInvalid: load data is invalid. 341 val data_valid = Bool() 342} 343 344class LoadNukeQueryResp(implicit p: Parameters) extends XSBundle { 345 // rep_frm_fetch: ld-ld violation check success, replay from fetch. 346 val rep_frm_fetch = Bool() 347} 348 349class LoadNukeQueryIO(implicit p: Parameters) extends XSBundle { 350 val req = Decoupled(new LoadNukeQueryReq) 351 val resp = Flipped(Valid(new LoadNukeQueryResp)) 352 val revoke = Output(Bool()) 353} 354 355class StoreNukeQueryIO(implicit p: Parameters) extends XSBundle { 356 // robIdx: Requestor's (a store instruction) rob index for match logic. 357 val robIdx = new RobPtr 358 359 // paddr: requestor's (a store instruction) physical address for match logic. 360 val paddr = UInt(PAddrBits.W) 361 362 // mask: requestor's (a store instruction) data width mask for match logic. 363 val mask = UInt((VLEN/8).W) 364 365 // matchLine: if store is vector 128-bits, load unit need to compare 128-bits vaddr. 366 val matchLine = Bool() 367} 368 369// Store byte valid mask write bundle 370// 371// Store byte valid mask write to SQ takes 2 cycles 372class StoreMaskBundle(implicit p: Parameters) extends XSBundle { 373 val sqIdx = new SqPtr 374 val mask = UInt((VLEN/8).W) 375} 376 377class LoadDataFromDcacheBundle(implicit p: Parameters) extends DCacheBundle { 378 // old dcache: optimize data sram read fanout 379 // val bankedDcacheData = Vec(DCacheBanks, UInt(64.W)) 380 // val bank_oh = UInt(DCacheBanks.W) 381 382 // new dcache 383 val respDcacheData = UInt(VLEN.W) 384 val forwardMask = Vec(VLEN/8, Bool()) 385 val forwardData = Vec(VLEN/8, UInt(8.W)) 386 val uop = new DynInst // for data selection, only fwen and fuOpType are used 387 val addrOffset = UInt(4.W) // for data selection 388 389 // forward tilelink D channel 390 val forward_D = Bool() 391 val forwardData_D = Vec(VLEN/8, UInt(8.W)) 392 393 // forward mshr data 394 val forward_mshr = Bool() 395 val forwardData_mshr = Vec(VLEN/8, UInt(8.W)) 396 397 val forward_result_valid = Bool() 398 399 def dcacheData(): UInt = { 400 // old dcache 401 // val dcache_data = Mux1H(bank_oh, bankedDcacheData) 402 // new dcache 403 val dcache_data = respDcacheData 404 val use_D = forward_D && forward_result_valid 405 val use_mshr = forward_mshr && forward_result_valid 406 Mux(use_D, forwardData_D.asUInt, Mux(use_mshr, forwardData_mshr.asUInt, dcache_data)) 407 } 408 409 def mergedData(): UInt = { 410 val rdataVec = VecInit((0 until VLEN / 8).map(j => 411 Mux(forwardMask(j), forwardData(j), dcacheData()(8*(j+1)-1, 8*j)) 412 )) 413 rdataVec.asUInt 414 } 415} 416 417// Load writeback data from load queue (refill) 418class LoadDataFromLQBundle(implicit p: Parameters) extends XSBundle { 419 val lqData = UInt(64.W) // load queue has merged data 420 val uop = new DynInst // for data selection, only fwen and fuOpType are used 421 val addrOffset = UInt(3.W) // for data selection 422 423 def mergedData(): UInt = { 424 lqData 425 } 426} 427 428// Bundle for load / store wait waking up 429class MemWaitUpdateReq(implicit p: Parameters) extends XSBundle { 430 val robIdx = Vec(backendParams.StaExuCnt, ValidIO(new RobPtr)) 431 val sqIdx = Vec(backendParams.StdCnt, ValidIO(new SqPtr)) 432} 433 434object AddPipelineReg { 435 class PipelineRegModule[T <: Data](gen: T) extends Module { 436 val io = IO(new Bundle() { 437 val in = Flipped(DecoupledIO(gen.cloneType)) 438 val out = DecoupledIO(gen.cloneType) 439 val isFlush = Input(Bool()) 440 }) 441 442 val valid = RegInit(false.B) 443 valid.suggestName("pipeline_reg_valid") 444 when (io.out.fire) { valid := false.B } 445 when (io.in.fire) { valid := true.B } 446 when (io.isFlush) { valid := false.B } 447 448 io.in.ready := !valid || io.out.ready 449 io.out.bits := RegEnable(io.in.bits, io.in.fire) 450 io.out.valid := valid //&& !isFlush 451 } 452 453 def apply[T <: Data] 454 (left: DecoupledIO[T], right: DecoupledIO[T], isFlush: Bool, 455 moduleName: Option[String] = None 456 ){ 457 val pipelineReg = Module(new PipelineRegModule[T](left.bits.cloneType)) 458 if(moduleName.nonEmpty) pipelineReg.suggestName(moduleName.get) 459 pipelineReg.io.in <> left 460 right <> pipelineReg.io.out 461 pipelineReg.io.isFlush := isFlush 462 } 463}