1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.fu.PMPRespBundle 27import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp} 28import xiangshan.v2backend.Bundles.{MemExuInput, MemExuOutput} 29import xiangshan.v2backend.{StaCfg} 30 31// Store Pipeline Stage 0 32// Generate addr, use addr to query DCache and DTLB 33class StoreUnit_S0(implicit p: Parameters) extends XSModule { 34 val io = IO(new Bundle() { 35 val in = Flipped(Decoupled(new MemExuInput)) 36 val rsIdx = Input(UInt(log2Up(IssQueSize).W)) 37 val isFirstIssue = Input(Bool()) 38 val out = Decoupled(new LsPipelineBundle) 39 val dtlbReq = DecoupledIO(new TlbReq) 40 }) 41 42 // send req to dtlb 43 // val saddr = io.in.bits.src(0) + SignExt(io.in.bits.uop.ctrl.imm(11,0), VAddrBits) 44 val imm12 = WireInit(io.in.bits.uop.imm(11,0)) 45 val saddr_lo = io.in.bits.src(0)(11,0) + Cat(0.U(1.W), imm12) 46 val saddr_hi = Mux(saddr_lo(12), 47 Mux(imm12(11), io.in.bits.src(0)(VAddrBits-1, 12), io.in.bits.src(0)(VAddrBits-1, 12)+1.U), 48 Mux(imm12(11), io.in.bits.src(0)(VAddrBits-1, 12)+SignExt(1.U, VAddrBits-12), io.in.bits.src(0)(VAddrBits-1, 12)), 49 ) 50 val saddr = Cat(saddr_hi, saddr_lo(11,0)) 51 52 io.dtlbReq.bits.vaddr := saddr 53 io.dtlbReq.valid := io.in.valid 54 io.dtlbReq.bits.cmd := TlbCmd.write 55 io.dtlbReq.bits.size := LSUOpType.size(io.in.bits.uop.fuOpType) 56 io.dtlbReq.bits.kill := DontCare 57 io.dtlbReq.bits.debug.robIdx := io.in.bits.uop.robIdx 58 io.dtlbReq.bits.debug.pc := io.in.bits.uop.pc 59 io.dtlbReq.bits.debug.isFirstIssue := io.isFirstIssue 60 61 io.out.bits := DontCare 62 io.out.bits.vaddr := saddr 63 64 // Now data use its own io 65 // io.out.bits.data := genWdata(io.in.bits.src(1), io.in.bits.uop.ctrl.fuOpType(1,0)) 66 io.out.bits.data := io.in.bits.src(1) // FIXME: remove data from pipeline 67 io.out.bits.uop := io.in.bits.uop 68 io.out.bits.miss := DontCare 69 io.out.bits.rsIdx := io.rsIdx 70 io.out.bits.mask := genWmask(io.out.bits.vaddr, io.in.bits.uop.fuOpType(1,0)) 71 io.out.bits.isFirstIssue := io.isFirstIssue 72 io.out.bits.wlineflag := io.in.bits.uop.fuOpType === LSUOpType.cbo_zero 73 io.out.valid := io.in.valid 74 io.in.ready := io.out.ready 75 76 // exception check 77 val addrAligned = LookupTree(io.in.bits.uop.fuOpType(1,0), List( 78 "b00".U -> true.B, //b 79 "b01".U -> (io.out.bits.vaddr(0) === 0.U), //h 80 "b10".U -> (io.out.bits.vaddr(1,0) === 0.U), //w 81 "b11".U -> (io.out.bits.vaddr(2,0) === 0.U) //d 82 )) 83 84 io.out.bits.uop.exceptionVec(storeAddrMisaligned) := !addrAligned 85 86 XSPerfAccumulate("in_valid", io.in.valid) 87 XSPerfAccumulate("in_fire", io.in.fire) 88 XSPerfAccumulate("in_fire_first_issue", io.in.fire && io.isFirstIssue) 89 XSPerfAccumulate("addr_spec_success", io.out.fire() && saddr(VAddrBits-1, 12) === io.in.bits.src(0)(VAddrBits-1, 12)) 90 XSPerfAccumulate("addr_spec_failed", io.out.fire() && saddr(VAddrBits-1, 12) =/= io.in.bits.src(0)(VAddrBits-1, 12)) 91 XSPerfAccumulate("addr_spec_success_once", io.out.fire() && saddr(VAddrBits-1, 12) === io.in.bits.src(0)(VAddrBits-1, 12) && io.isFirstIssue) 92 XSPerfAccumulate("addr_spec_failed_once", io.out.fire() && saddr(VAddrBits-1, 12) =/= io.in.bits.src(0)(VAddrBits-1, 12) && io.isFirstIssue) 93} 94 95// Store Pipeline Stage 1 96// TLB resp (send paddr to dcache) 97class StoreUnit_S1(implicit p: Parameters) extends XSModule { 98 val io = IO(new Bundle() { 99 val in = Flipped(Decoupled(new LsPipelineBundle)) 100 val out = Decoupled(new LsPipelineBundle) 101 val lsq = ValidIO(new LsPipelineBundle()) 102 val dtlbResp = Flipped(DecoupledIO(new TlbResp())) 103 val rsFeedback = ValidIO(new RSFeedback) 104 val reExecuteQuery = Valid(new LoadReExecuteQueryIO) 105 }) 106 107 // mmio cbo decoder 108 val is_mmio_cbo = io.in.bits.uop.fuOpType === LSUOpType.cbo_clean || 109 io.in.bits.uop.fuOpType === LSUOpType.cbo_flush || 110 io.in.bits.uop.fuOpType === LSUOpType.cbo_inval 111 112 val s1_paddr = io.dtlbResp.bits.paddr(0) 113 val s1_tlb_miss = io.dtlbResp.bits.miss 114 val s1_mmio = is_mmio_cbo 115 val s1_exception = ExceptionNO.selectByFu(io.out.bits.uop.exceptionVec, StaCfg).asUInt.orR 116 117 io.in.ready := true.B 118 119 io.dtlbResp.ready := true.B // TODO: why dtlbResp needs a ready? 120 121 // st-ld violation dectect request. 122 io.reExecuteQuery.valid := io.in.valid && !s1_tlb_miss 123 io.reExecuteQuery.bits.robIdx := io.in.bits.uop.robIdx 124 io.reExecuteQuery.bits.paddr := s1_paddr 125 io.reExecuteQuery.bits.mask := io.in.bits.mask 126 127 // Send TLB feedback to store issue queue 128 // Store feedback is generated in store_s1, sent to RS in store_s2 129 io.rsFeedback.valid := io.in.valid 130 io.rsFeedback.bits.hit := !s1_tlb_miss 131 io.rsFeedback.bits.flushState := io.dtlbResp.bits.ptwBack 132 io.rsFeedback.bits.rsIdx := io.in.bits.rsIdx 133 io.rsFeedback.bits.sourceType := RSFeedbackType.tlbMiss 134 XSDebug(io.rsFeedback.valid, 135 "S1 Store: tlbHit: %d robIdx: %d\n", 136 io.rsFeedback.bits.hit, 137 io.rsFeedback.bits.rsIdx 138 ) 139 io.rsFeedback.bits.dataInvalidSqIdx := DontCare 140 141 // get paddr from dtlb, check if rollback is needed 142 // writeback store inst to lsq 143 io.out.valid := io.in.valid && !s1_tlb_miss 144 io.out.bits := io.in.bits 145 io.out.bits.paddr := s1_paddr 146 io.out.bits.miss := false.B 147 io.out.bits.mmio := s1_mmio 148 io.out.bits.atomic := s1_mmio 149 io.out.bits.uop.exceptionVec(storePageFault) := io.dtlbResp.bits.excp(0).pf.st 150 io.out.bits.uop.exceptionVec(storeAccessFault) := io.dtlbResp.bits.excp(0).af.st 151 152 io.lsq.valid := io.in.valid 153 io.lsq.bits := io.out.bits 154 io.lsq.bits.miss := s1_tlb_miss 155 156 // mmio inst with exception will be writebacked immediately 157 // io.out.valid := io.in.valid && (!io.out.bits.mmio || s1_exception) && !s1_tlb_miss 158 159 XSPerfAccumulate("in_valid", io.in.valid) 160 XSPerfAccumulate("in_fire", io.in.fire) 161 XSPerfAccumulate("in_fire_first_issue", io.in.fire && io.in.bits.isFirstIssue) 162 XSPerfAccumulate("tlb_miss", io.in.fire && s1_tlb_miss) 163 XSPerfAccumulate("tlb_miss_first_issue", io.in.fire && s1_tlb_miss && io.in.bits.isFirstIssue) 164} 165 166class StoreUnit_S2(implicit p: Parameters) extends XSModule { 167 val io = IO(new Bundle() { 168 val in = Flipped(Decoupled(new LsPipelineBundle)) 169 val pmpResp = Flipped(new PMPRespBundle) 170 val static_pm = Input(Valid(Bool())) 171 val out = Decoupled(new LsPipelineBundle) 172 }) 173 val pmp = WireInit(io.pmpResp) 174 when (io.static_pm.valid) { 175 pmp.ld := false.B 176 pmp.st := false.B 177 pmp.instr := false.B 178 pmp.mmio := io.static_pm.bits 179 } 180 181 val s2_exception = ExceptionNO.selectByFu(io.out.bits.uop.exceptionVec, StaCfg).asUInt.orR 182 val is_mmio = io.in.bits.mmio || pmp.mmio 183 184 io.in.ready := true.B 185 io.out.bits := io.in.bits 186 io.out.bits.mmio := is_mmio && !s2_exception 187 io.out.bits.atomic := io.in.bits.atomic || pmp.atomic 188 io.out.bits.uop.exceptionVec(storeAccessFault) := io.in.bits.uop.exceptionVec(storeAccessFault) || pmp.st 189 io.out.valid := io.in.valid && (!is_mmio || s2_exception) 190} 191 192class StoreUnit_S3(implicit p: Parameters) extends XSModule { 193 val io = IO(new Bundle() { 194 val in = Flipped(Decoupled(new LsPipelineBundle)) 195 val stout = DecoupledIO(new MemExuOutput) // writeback store 196 }) 197 198 io.in.ready := true.B 199 200 io.stout.valid := io.in.valid 201 io.stout.bits.uop := io.in.bits.uop 202 io.stout.bits.data := DontCare 203 io.stout.bits.debug.isMMIO := io.in.bits.mmio 204 io.stout.bits.debug.paddr := io.in.bits.paddr 205 io.stout.bits.debug.vaddr := io.in.bits.vaddr 206 io.stout.bits.debug.isPerfCnt := false.B 207} 208 209class StoreUnit(implicit p: Parameters) extends XSModule { 210 val io = IO(new Bundle() { 211 val stin = Flipped(Decoupled(new MemExuInput)) 212 val redirect = Flipped(ValidIO(new Redirect)) 213 val feedbackSlow = ValidIO(new RSFeedback) 214 val tlb = new TlbRequestIO() 215 val pmp = Flipped(new PMPRespBundle()) 216 val rsIdx = Input(UInt(log2Up(IssQueSize).W)) 217 val isFirstIssue = Input(Bool()) 218 val lsq = ValidIO(new LsPipelineBundle) 219 val lsq_replenish = Output(new LsPipelineBundle()) 220 val stout = DecoupledIO(new MemExuOutput) // writeback store 221 // store mask, send to sq in store_s0 222 val storeMaskOut = Valid(new StoreMaskBundle) 223 val reExecuteQuery = Valid(new LoadReExecuteQueryIO) 224 val issue = Valid(new MemExuInput) 225 }) 226 227 val store_s0 = Module(new StoreUnit_S0) 228 val store_s1 = Module(new StoreUnit_S1) 229 val store_s2 = Module(new StoreUnit_S2) 230 val store_s3 = Module(new StoreUnit_S3) 231 232 store_s0.io.in <> io.stin 233 store_s0.io.dtlbReq <> io.tlb.req 234 io.tlb.req_kill := false.B 235 store_s0.io.rsIdx := io.rsIdx 236 store_s0.io.isFirstIssue := io.isFirstIssue 237 238 io.storeMaskOut.valid := store_s0.io.in.valid 239 io.storeMaskOut.bits.mask := store_s0.io.out.bits.mask 240 io.storeMaskOut.bits.sqIdx := store_s0.io.out.bits.uop.sqIdx 241 242 PipelineConnect(store_s0.io.out, store_s1.io.in, true.B, store_s0.io.out.bits.uop.robIdx.needFlush(io.redirect)) 243 io.issue.valid := store_s1.io.in.valid && !store_s1.io.dtlbResp.bits.miss 244 io.issue.bits := RegEnable(store_s0.io.in.bits, store_s0.io.in.valid) 245 246 store_s1.io.dtlbResp <> io.tlb.resp 247 io.lsq <> store_s1.io.lsq 248 io.reExecuteQuery := store_s1.io.reExecuteQuery 249 250 PipelineConnect(store_s1.io.out, store_s2.io.in, true.B, store_s1.io.out.bits.uop.robIdx.needFlush(io.redirect)) 251 252 // feedback tlb miss to RS in store_s2 253 io.feedbackSlow.bits := RegNext(store_s1.io.rsFeedback.bits) 254 io.feedbackSlow.valid := RegNext(store_s1.io.rsFeedback.valid && !store_s1.io.out.bits.uop.robIdx.needFlush(io.redirect)) 255 256 store_s2.io.pmpResp <> io.pmp 257 store_s2.io.static_pm := RegNext(io.tlb.resp.bits.static_pm) 258 io.lsq_replenish := store_s2.io.out.bits // mmio and exception 259 PipelineConnect(store_s2.io.out, store_s3.io.in, true.B, store_s2.io.out.bits.uop.robIdx.needFlush(io.redirect)) 260 261 store_s3.io.stout <> io.stout 262 263 private def printPipeLine(pipeline: LsPipelineBundle, cond: Bool, name: String): Unit = { 264 XSDebug(cond, 265 p"$name" + p" pc ${Hexadecimal(pipeline.uop.pc)} " + 266 p"addr ${Hexadecimal(pipeline.vaddr)} -> ${Hexadecimal(pipeline.paddr)} " + 267 p"op ${Binary(pipeline.uop.fuOpType)} " + 268 p"data ${Hexadecimal(pipeline.data)} " + 269 p"mask ${Hexadecimal(pipeline.mask)}\n" 270 ) 271 } 272 273 printPipeLine(store_s0.io.out.bits, store_s0.io.out.valid, "S0") 274 printPipeLine(store_s1.io.out.bits, store_s1.io.out.valid, "S1") 275} 276