1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19 20import chipsalliance.rocketchip.config.Parameters 21import chisel3._ 22import chisel3.util._ 23import xiangshan._ 24import utils._ 25import utility._ 26import xiangshan.backend.rob.RobPtr 27import xiangshan.cache._ 28import xiangshan.backend.fu.FenceToSbuffer 29import xiangshan.cache.dcache.ReplayCarry 30 31object genWmask { 32 def apply(addr: UInt, sizeEncode: UInt): UInt = { 33 (LookupTree(sizeEncode, List( 34 "b00".U -> 0x1.U, //0001 << addr(2:0) 35 "b01".U -> 0x3.U, //0011 36 "b10".U -> 0xf.U, //1111 37 "b11".U -> 0xff.U //11111111 38 )) << addr(2, 0)).asUInt() 39 } 40} 41 42object genWdata { 43 def apply(data: UInt, sizeEncode: UInt): UInt = { 44 LookupTree(sizeEncode, List( 45 "b00".U -> Fill(8, data(7, 0)), 46 "b01".U -> Fill(4, data(15, 0)), 47 "b10".U -> Fill(2, data(31, 0)), 48 "b11".U -> data 49 )) 50 } 51} 52 53class LsPipelineBundle(implicit p: Parameters) extends XSBundleWithMicroOp with HasDCacheParameters{ 54 val vaddr = UInt(VAddrBits.W) 55 val paddr = UInt(PAddrBits.W) 56 // val func = UInt(6.W) 57 val mask = UInt(8.W) 58 val data = UInt((XLEN+1).W) 59 val wlineflag = Bool() // store write the whole cache line 60 61 val miss = Bool() 62 val tlbMiss = Bool() 63 val ptwBack = Bool() 64 val mmio = Bool() 65 val atomic = Bool() 66 val rsIdx = UInt(log2Up(IssQueSize).W) 67 68 val forwardMask = Vec(8, Bool()) 69 val forwardData = Vec(8, UInt(8.W)) 70 71 // prefetch 72 val isPrefetch = Bool() 73 val isHWPrefetch = Bool() 74 75 // For debug usage 76 val isFirstIssue = Bool() 77 78 // For load replay 79 val isLoadReplay = Bool() 80 val replayCarry = new ReplayCarry 81 82 // For dcache miss load 83 val mshrid = UInt(log2Up(cfg.nMissEntries).W) 84 85 val forward_tlDchannel = Bool() 86} 87 88class LdPrefetchTrainBundle(implicit p: Parameters) extends LsPipelineBundle { 89 val meta_prefetch = Bool() 90 val meta_access = Bool() 91 92 def fromLsPipelineBundle(input: LsPipelineBundle) = { 93 vaddr := input.vaddr 94 paddr := input.paddr 95 mask := input.mask 96 data := input.data 97 uop := input.uop 98 wlineflag := input.wlineflag 99 miss := input.miss 100 tlbMiss := input.tlbMiss 101 ptwBack := input.ptwBack 102 mmio := input.mmio 103 rsIdx := input.rsIdx 104 forwardMask := input.forwardMask 105 forwardData := input.forwardData 106 isPrefetch := input.isPrefetch 107 isHWPrefetch := input.isHWPrefetch 108 isFirstIssue := input.isFirstIssue 109 meta_prefetch := DontCare 110 meta_access := DontCare 111 forward_tlDchannel := DontCare 112 mshrid := DontCare 113 replayCarry := DontCare 114 atomic := DontCare 115 isLoadReplay := DontCare 116 } 117} 118 119class LqWriteBundle(implicit p: Parameters) extends LsPipelineBundle { 120 // queue entry data, except flag bits, will be updated if writeQueue is true, 121 // valid bit in LqWriteBundle will be ignored 122 val lq_data_wen_dup = Vec(6, Bool()) // dirty reg dup 123 124 def fromLsPipelineBundle(input: LsPipelineBundle) = { 125 vaddr := input.vaddr 126 paddr := input.paddr 127 mask := input.mask 128 data := input.data 129 uop := input.uop 130 wlineflag := input.wlineflag 131 miss := input.miss 132 tlbMiss := input.tlbMiss 133 ptwBack := input.ptwBack 134 mmio := input.mmio 135 atomic := input.atomic 136 rsIdx := input.rsIdx 137 forwardMask := input.forwardMask 138 forwardData := input.forwardData 139 isPrefetch := input.isPrefetch 140 isHWPrefetch := input.isHWPrefetch 141 isFirstIssue := input.isFirstIssue 142 isLoadReplay := input.isLoadReplay 143 mshrid := input.mshrid 144 forward_tlDchannel := input.forward_tlDchannel 145 replayCarry := input.replayCarry 146 147 lq_data_wen_dup := DontCare 148 } 149} 150 151class LoadForwardQueryIO(implicit p: Parameters) extends XSBundleWithMicroOp { 152 val vaddr = Output(UInt(VAddrBits.W)) 153 val paddr = Output(UInt(PAddrBits.W)) 154 val mask = Output(UInt(8.W)) 155 override val uop = Output(new MicroOp) // for replay 156 val pc = Output(UInt(VAddrBits.W)) //for debug 157 val valid = Output(Bool()) 158 159 val forwardMaskFast = Input(Vec(8, Bool())) // resp to load_s1 160 val forwardMask = Input(Vec(8, Bool())) // resp to load_s2 161 val forwardData = Input(Vec(8, UInt(8.W))) // resp to load_s2 162 163 // val lqIdx = Output(UInt(LoadQueueIdxWidth.W)) 164 val sqIdx = Output(new SqPtr) 165 166 // dataInvalid suggests store to load forward found forward should happen, 167 // but data is not available for now. If dataInvalid, load inst should 168 // be replayed from RS. Feedback type should be RSFeedbackType.dataInvalid 169 val dataInvalid = Input(Bool()) // Addr match, but data is not valid for now 170 171 // matchInvalid suggests in store to load forward logic, paddr cam result does 172 // to equal to vaddr cam result. If matchInvalid, a microarchitectural exception 173 // should be raised to flush SQ and committed sbuffer. 174 val matchInvalid = Input(Bool()) // resp to load_s2 175} 176 177// LoadForwardQueryIO used in load pipeline 178// 179// Difference between PipeLoadForwardQueryIO and LoadForwardQueryIO: 180// PipeIO use predecoded sqIdxMask for better forward timing 181class PipeLoadForwardQueryIO(implicit p: Parameters) extends LoadForwardQueryIO { 182 // val sqIdx = Output(new SqPtr) // for debug, should not be used in pipeline for timing reasons 183 // sqIdxMask is calcuated in earlier stage for better timing 184 val sqIdxMask = Output(UInt(StoreQueueSize.W)) 185 186 // dataInvalid: addr match, but data is not valid for now 187 val dataInvalidFast = Input(Bool()) // resp to load_s1 188 // val dataInvalid = Input(Bool()) // resp to load_s2 189 val dataInvalidSqIdx = Input(UInt(log2Up(StoreQueueSize).W)) // resp to load_s2, sqIdx value 190} 191 192// Query load queue for ld-ld violation 193// 194// Req should be send in load_s1 195// Resp will be generated 1 cycle later 196// 197// Note that query req may be !ready, as dcache is releasing a block 198// If it happens, a replay from rs is needed. 199 200class LoadViolationQueryReq(implicit p: Parameters) extends XSBundleWithMicroOp { // provide lqIdx 201 val paddr = UInt(PAddrBits.W) 202} 203 204class LoadViolationQueryResp(implicit p: Parameters) extends XSBundle { 205 val have_violation = Bool() 206} 207 208class LoadViolationQueryIO(implicit p: Parameters) extends XSBundle { 209 val req = Decoupled(new LoadViolationQueryReq) 210 val resp = Flipped(Valid(new LoadViolationQueryResp)) 211} 212 213class LoadReExecuteQueryIO(implicit p: Parameters) extends XSBundle { 214 // robIdx: Requestor's (a store instruction) rob index for match logic. 215 val robIdx = new RobPtr 216 217 // paddr: requestor's (a store instruction) physical address for match logic. 218 val paddr = UInt(PAddrBits.W) 219 220 // mask: requestor's (a store instruction) data width mask for match logic. 221 val mask = UInt(8.W) 222} 223 224// Store byte valid mask write bundle 225// 226// Store byte valid mask write to SQ takes 2 cycles 227class StoreMaskBundle(implicit p: Parameters) extends XSBundle { 228 val sqIdx = new SqPtr 229 val mask = UInt(8.W) 230} 231 232class LoadDataFromDcacheBundle(implicit p: Parameters) extends DCacheBundle { 233 // old dcache: optimize data sram read fanout 234 // val bankedDcacheData = Vec(DCacheBanks, UInt(64.W)) 235 // val bank_oh = UInt(DCacheBanks.W) 236 237 // new dcache 238 val respDcacheData = UInt(XLEN.W) 239 val forwardMask = Vec(8, Bool()) 240 val forwardData = Vec(8, UInt(8.W)) 241 val uop = new MicroOp // for data selection, only fwen and fuOpType are used 242 val addrOffset = UInt(3.W) // for data selection 243 244 // forward tilelink D channel 245 val forward_D = Input(Bool()) 246 val forwardData_D = Input(Vec(8, UInt(8.W))) 247 248 // forward mshr data 249 val forward_mshr = Input(Bool()) 250 val forwardData_mshr = Input(Vec(8, UInt(8.W))) 251 252 val forward_result_valid = Input(Bool()) 253 254 def dcacheData(): UInt = { 255 // old dcache 256 // val dcache_data = Mux1H(bank_oh, bankedDcacheData) 257 // new dcache 258 val dcache_data = respDcacheData 259 val use_D = forward_D && forward_result_valid 260 val use_mshr = forward_mshr && forward_result_valid 261 Mux(use_D, forwardData_D.asUInt, Mux(use_mshr, forwardData_mshr.asUInt, dcache_data)) 262 } 263 264 def mergedData(): UInt = { 265 val rdataVec = VecInit((0 until XLEN / 8).map(j => 266 Mux(forwardMask(j), forwardData(j), dcacheData()(8*(j+1)-1, 8*j)) 267 )) 268 rdataVec.asUInt 269 } 270} 271 272// Load writeback data from load queue (refill) 273class LoadDataFromLQBundle(implicit p: Parameters) extends XSBundle { 274 val lqData = UInt(64.W) // load queue has merged data 275 val uop = new MicroOp // for data selection, only fwen and fuOpType are used 276 val addrOffset = UInt(3.W) // for data selection 277 278 def mergedData(): UInt = { 279 lqData 280 } 281} 282 283// Bundle for load / store wait waking up 284class MemWaitUpdateReq(implicit p: Parameters) extends XSBundle { 285 val staIssue = Vec(exuParameters.StuCnt, ValidIO(new ExuInput)) 286 val stdIssue = Vec(exuParameters.StuCnt, ValidIO(new ExuInput)) 287} 288 289object AddPipelineReg { 290 class PipelineRegModule[T <: Data](gen: T) extends Module { 291 val io = IO(new Bundle() { 292 val in = Flipped(DecoupledIO(gen.cloneType)) 293 val out = DecoupledIO(gen.cloneType) 294 val isFlush = Input(Bool()) 295 }) 296 297 val valid = RegInit(false.B) 298 valid.suggestName("pipeline_reg_valid") 299 when (io.out.fire()) { valid := false.B } 300 when (io.in.fire()) { valid := true.B } 301 when (io.isFlush) { valid := false.B } 302 303 io.in.ready := !valid || io.out.ready 304 io.out.bits := RegEnable(io.in.bits, io.in.fire()) 305 io.out.valid := valid //&& !isFlush 306 } 307 308 def apply[T <: Data] 309 (left: DecoupledIO[T], right: DecoupledIO[T], isFlush: Bool, 310 moduleName: Option[String] = None 311 ){ 312 val pipelineReg = Module(new PipelineRegModule[T](left.bits.cloneType)) 313 if(moduleName.nonEmpty) pipelineReg.suggestName(moduleName.get) 314 pipelineReg.io.in <> left 315 right <> pipelineReg.io.out 316 pipelineReg.io.isFlush := isFlush 317 } 318} 319