1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.fu.PMPRespBundle 27import xiangshan.backend.rob.DebugLsInfoBundle 28import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp} 29 30class StoreUnit(implicit p: Parameters) extends XSModule { 31 val io = IO(new Bundle() { 32 val redirect = Flipped(ValidIO(new Redirect)) 33 val stin = Flipped(Decoupled(new ExuInput)) 34 val issue = Valid(new ExuInput) 35 val tlb = new TlbRequestIO() 36 val pmp = Flipped(new PMPRespBundle()) 37 val rsIdx = Input(UInt(log2Up(IssQueSize).W)) 38 val isFirstIssue = Input(Bool()) 39 val lsq = ValidIO(new LsPipelineBundle) 40 val lsq_replenish = Output(new LsPipelineBundle()) 41 val feedback_slow = ValidIO(new RSFeedback) 42 val stld_nuke_query = Valid(new StoreNukeQueryIO) 43 val stout = DecoupledIO(new ExuOutput) // writeback store 44 // store mask, send to sq in store_s0 45 val st_mask_out = Valid(new StoreMaskBundle) 46 val debug_ls = Output(new DebugLsInfoBundle) 47 }) 48 49 val s1_ready, s2_ready, s3_ready = WireInit(false.B) 50 51 // Pipeline 52 // -------------------------------------------------------------------------------- 53 // stage 0 54 // -------------------------------------------------------------------------------- 55 // generate addr, use addr to query DCache and DTLB 56 val s0_valid = io.stin.valid 57 val s0_in = io.stin.bits 58 val s0_isFirstIssue = io.isFirstIssue 59 val s0_rsIdx = io.rsIdx 60 val s0_out = Wire(new LsPipelineBundle) 61 val s0_kill = s0_in.uop.robIdx.needFlush(io.redirect) 62 val s0_can_go = s1_ready 63 val s0_fire = s0_valid && !s0_kill && s0_can_go 64 65 // generate addr 66 // val saddr = s0_in.bits.src(0) + SignExt(s0_in.bits.uop.ctrl.imm(11,0), VAddrBits) 67 val imm12 = WireInit(s0_in.uop.ctrl.imm(11,0)) 68 val saddr_lo = s0_in.src(0)(11,0) + Cat(0.U(1.W), imm12) 69 val saddr_hi = Mux(saddr_lo(12), 70 Mux(imm12(11), s0_in.src(0)(VAddrBits-1, 12), s0_in.src(0)(VAddrBits-1, 12)+1.U), 71 Mux(imm12(11), s0_in.src(0)(VAddrBits-1, 12)+SignExt(1.U, VAddrBits-12), s0_in.src(0)(VAddrBits-1, 12)), 72 ) 73 val s0_saddr = Cat(saddr_hi, saddr_lo(11,0)) 74 75 io.tlb.req.valid := s0_valid 76 io.tlb.req.bits.vaddr := s0_saddr 77 io.tlb.req.bits.cmd := TlbCmd.write 78 io.tlb.req.bits.size := LSUOpType.size(s0_in.uop.ctrl.fuOpType) 79 io.tlb.req.bits.kill := DontCare 80 io.tlb.req.bits.memidx.is_ld := false.B 81 io.tlb.req.bits.memidx.is_st := true.B 82 io.tlb.req.bits.memidx.idx := s0_in.uop.sqIdx.value 83 io.tlb.req.bits.debug.robIdx := s0_in.uop.robIdx 84 io.tlb.req.bits.no_translate := false.B 85 io.tlb.req.bits.debug.pc := s0_in.uop.cf.pc 86 io.tlb.req.bits.debug.isFirstIssue := s0_isFirstIssue 87 io.tlb.req_kill := false.B 88 89 s0_out := DontCare 90 s0_out.vaddr := s0_saddr 91 // Now data use its own io 92 // s1_out.data := genWdata(s1_in.src(1), s1_in.uop.ctrl.fuOpType(1,0)) 93 s0_out.data := s0_in.src(1) // FIXME: remove data from pipeline 94 s0_out.uop := s0_in.uop 95 s0_out.miss := DontCare 96 s0_out.rsIdx := s0_rsIdx 97 s0_out.mask := genVWmask(s0_saddr, s0_in.uop.ctrl.fuOpType(1,0)) 98 s0_out.isFirstIssue := s0_isFirstIssue 99 s0_out.isHWPrefetch := false.B // TODO 100 s0_out.wlineflag := s0_in.uop.ctrl.fuOpType === LSUOpType.cbo_zero 101 when(s0_valid && s0_isFirstIssue) { 102 s0_out.uop.debugInfo.tlbFirstReqTime := GTimer() 103 } 104 105 // exception check 106 val s0_addr_aligned = LookupTree(s0_in.uop.ctrl.fuOpType(1,0), List( 107 "b00".U -> true.B, //b 108 "b01".U -> (s0_out.vaddr(0) === 0.U), //h 109 "b10".U -> (s0_out.vaddr(1,0) === 0.U), //w 110 "b11".U -> (s0_out.vaddr(2,0) === 0.U) //d 111 )) 112 s0_out.uop.cf.exceptionVec(storeAddrMisaligned) := !s0_addr_aligned 113 114 io.st_mask_out.valid := s0_valid 115 io.st_mask_out.bits.mask := s0_out.mask 116 io.st_mask_out.bits.sqIdx := s0_out.uop.sqIdx 117 118 io.stin.ready := s1_ready 119 120 // Pipeline 121 // -------------------------------------------------------------------------------- 122 // stage 1 123 // -------------------------------------------------------------------------------- 124 // TLB resp (send paddr to dcache) 125 val s1_valid = RegInit(false.B) 126 val s1_in = RegEnable(s0_out, s0_fire) 127 val s1_out = Wire(new LsPipelineBundle) 128 val s1_kill = Wire(Bool()) 129 val s1_can_go = s2_ready 130 val s1_fire = s1_valid && !s1_kill && s1_can_go 131 132 // mmio cbo decoder 133 val s1_mmio_cbo = s1_in.uop.ctrl.fuOpType === LSUOpType.cbo_clean || 134 s1_in.uop.ctrl.fuOpType === LSUOpType.cbo_flush || 135 s1_in.uop.ctrl.fuOpType === LSUOpType.cbo_inval 136 val s1_paddr = io.tlb.resp.bits.paddr(0) 137 val s1_tlb_miss = io.tlb.resp.bits.miss 138 val s1_mmio = s1_mmio_cbo 139 val s1_exception = ExceptionNO.selectByFu(s1_out.uop.cf.exceptionVec, staCfg).asUInt.orR 140 s1_kill := s1_in.uop.robIdx.needFlush(io.redirect) || s1_tlb_miss 141 142 s1_ready := !s1_valid || s1_kill || s2_ready 143 io.tlb.resp.ready := true.B // TODO: why dtlbResp needs a ready? 144 when (s0_fire) { s1_valid := true.B } 145 .elsewhen (s1_fire) { s1_valid := false.B } 146 .elsewhen (s1_kill) { s1_valid := false.B } 147 148 // st-ld violation dectect request. 149 io.stld_nuke_query.valid := s1_valid && !s1_tlb_miss 150 io.stld_nuke_query.bits.robIdx := s1_in.uop.robIdx 151 io.stld_nuke_query.bits.paddr := s1_paddr 152 io.stld_nuke_query.bits.mask := s1_in.mask 153 154 // Send TLB feedback to store issue queue 155 // Store feedback is generated in store_s1, sent to RS in store_s2 156 io.feedback_slow.valid := s1_fire 157 io.feedback_slow.bits.hit := !s1_tlb_miss 158 io.feedback_slow.bits.flushState := io.tlb.resp.bits.ptwBack 159 io.feedback_slow.bits.rsIdx := s1_in.rsIdx 160 io.feedback_slow.bits.sourceType := RSFeedbackType.tlbMiss 161 XSDebug(io.feedback_slow.valid, 162 "S1 Store: tlbHit: %d robIdx: %d\n", 163 io.feedback_slow.bits.hit, 164 io.feedback_slow.bits.rsIdx 165 ) 166 io.feedback_slow.bits.dataInvalidSqIdx := DontCare 167 168 // issue 169 io.issue.valid := s1_valid && !s1_tlb_miss 170 io.issue.bits := RegEnable(s0_in, s0_valid) 171 172 // rs feedback 173 val s1_feedback = Wire(Valid(new RSFeedback)) 174 s1_feedback.valid := s1_valid & !s1_in.isHWPrefetch 175 s1_feedback.bits.hit := !s1_tlb_miss 176 s1_feedback.bits.flushState := io.tlb.resp.bits.ptwBack 177 s1_feedback.bits.rsIdx := s1_out.rsIdx 178 s1_feedback.bits.sourceType := RSFeedbackType.tlbMiss 179 s1_feedback.bits.dataInvalidSqIdx := DontCare 180 XSDebug(s1_feedback.valid, 181 "S1 Store: tlbHit: %d robIdx: %d\n", 182 s1_feedback.bits.hit, 183 s1_feedback.bits.rsIdx 184 ) 185 186 // get paddr from dtlb, check if rollback is needed 187 // writeback store inst to lsq 188 s1_out := s1_in 189 s1_out.paddr := s1_paddr 190 s1_out.miss := false.B 191 s1_out.mmio := s1_mmio 192 s1_out.atomic := s1_mmio 193 s1_out.uop.cf.exceptionVec(storePageFault) := io.tlb.resp.bits.excp(0).pf.st 194 s1_out.uop.cf.exceptionVec(storeAccessFault) := io.tlb.resp.bits.excp(0).af.st 195 196 io.lsq.valid := s1_valid 197 io.lsq.bits := s1_out 198 io.lsq.bits.miss := s1_tlb_miss 199 200 // write below io.out.bits assign sentence to prevent overwriting values 201 val s1_tlb_memidx = io.tlb.resp.bits.memidx 202 when(s1_tlb_memidx.is_st && io.tlb.resp.valid && !s1_tlb_miss && s1_tlb_memidx.idx === s1_out.uop.sqIdx.value) { 203 // printf("Store idx = %d\n", s1_tlb_memidx.idx) 204 s1_out.uop.debugInfo.tlbRespTime := GTimer() 205 } 206 207 // Pipeline 208 // -------------------------------------------------------------------------------- 209 // stage 2 210 // -------------------------------------------------------------------------------- 211 // mmio check 212 val s2_valid = RegInit(false.B) 213 val s2_in = RegEnable(s1_out, s1_fire) 214 val s2_out = Wire(new LsPipelineBundle) 215 val s2_kill = Wire(Bool()) 216 val s2_can_go = s3_ready 217 val s2_fire = s2_valid && !s2_kill && s2_can_go 218 219 s2_ready := !s2_valid || s2_kill || s3_ready 220 when (s1_fire) { s2_valid := true.B } 221 .elsewhen (s2_fire) { s2_valid := false.B } 222 .elsewhen (s2_kill) { s2_valid := false.B } 223 224 val s2_pmp = WireInit(io.pmp) 225 val s2_static_pm = RegNext(io.tlb.resp.bits.static_pm) 226 when (s2_static_pm.valid) { 227 s2_pmp.ld := false.B 228 s2_pmp.st := false.B 229 s2_pmp.instr := false.B 230 s2_pmp.mmio := s2_static_pm.bits 231 } 232 233 val s2_exception = ExceptionNO.selectByFu(s2_out.uop.cf.exceptionVec, staCfg).asUInt.orR 234 val s2_mmio = s2_in.mmio || s2_pmp.mmio 235 s2_kill := (s2_mmio && !s2_exception) || s2_in.uop.robIdx.needFlush(io.redirect) 236 237 s2_out := s2_in 238 s2_out.mmio := s2_mmio && !s2_exception 239 s2_out.atomic := s2_in.atomic || s2_pmp.atomic 240 s2_out.uop.cf.exceptionVec(storeAccessFault) := s2_in.uop.cf.exceptionVec(storeAccessFault) || s2_pmp.st 241 242 // feedback tlb miss to RS in store_s2 243 io.feedback_slow.valid := RegNext(s1_feedback.valid && !s1_out.uop.robIdx.needFlush(io.redirect)) 244 io.feedback_slow.bits := RegNext(s1_feedback.bits) 245 246 // mmio and exception 247 io.lsq_replenish := s2_out 248 249 // Pipeline 250 // -------------------------------------------------------------------------------- 251 // stage 3 252 // -------------------------------------------------------------------------------- 253 // store write back 254 val s3_valid = RegInit(false.B) 255 val s3_in = RegEnable(s2_out, s2_fire) 256 val s3_out = Wire(new ExuOutput) 257 val s3_kill = s3_in.uop.robIdx.needFlush(io.redirect) 258 val s3_can_go = s3_ready 259 val s3_fire = s3_valid && !s3_kill && s3_can_go 260 261 when (s2_fire) { s3_valid := (!s2_mmio || s2_exception) && !s2_out.isHWPrefetch } 262 .elsewhen (s3_fire) { s3_valid := false.B } 263 .elsewhen (s3_kill) { s3_valid := false.B } 264 265 // wb: writeback 266 val SelectGroupSize = RollbackGroupSize 267 val lgSelectGroupSize = log2Ceil(SelectGroupSize) 268 val TotalSelectCycles = scala.math.ceil(log2Ceil(LoadQueueRAWSize).toFloat / lgSelectGroupSize).toInt + 1 269 270 s3_out := DontCare 271 s3_out.uop := s3_in.uop 272 s3_out.data := DontCare 273 s3_out.redirectValid := false.B 274 s3_out.redirect := DontCare 275 s3_out.debug.isMMIO := s3_in.mmio 276 s3_out.debug.paddr := s3_in.paddr 277 s3_out.debug.vaddr := s3_in.vaddr 278 s3_out.debug.isPerfCnt := false.B 279 s3_out.fflags := DontCare 280 281 // Pipeline 282 // -------------------------------------------------------------------------------- 283 // stage x 284 // -------------------------------------------------------------------------------- 285 // delay TotalSelectCycles - 2 cycle(s) 286 val TotalDelayCycles = TotalSelectCycles - 2 287 val sx_valid = Wire(Vec(TotalDelayCycles + 1, Bool())) 288 val sx_ready = Wire(Vec(TotalDelayCycles + 1, Bool())) 289 val sx_in = Wire(Vec(TotalDelayCycles + 1, new ExuOutput)) 290 291 // backward ready signal 292 s3_ready := sx_ready.head 293 for (i <- 0 until TotalDelayCycles + 1) { 294 if (i == 0) { 295 sx_valid(i) := s3_valid 296 sx_in(i) := s3_out 297 sx_ready(i) := !s3_valid(i) || sx_in(i).uop.robIdx.needFlush(io.redirect) || (if (TotalDelayCycles == 0) io.stout.ready else sx_ready(i+1)) 298 } else { 299 val cur_kill = sx_in(i).uop.robIdx.needFlush(io.redirect) 300 val cur_can_go = (if (i == TotalDelayCycles) io.stout.ready else sx_ready(i+1)) 301 val cur_fire = sx_valid(i) && !cur_kill && cur_can_go 302 val prev_fire = sx_valid(i-1) && !sx_in(i-1).uop.robIdx.needFlush(io.redirect) && sx_ready(i) 303 304 sx_ready(i) := !sx_valid(i) || cur_kill || (if (i == TotalDelayCycles) io.stout.ready else sx_ready(i+1)) 305 val sx_valid_can_go = prev_fire || cur_fire || cur_kill 306 sx_valid(i) := RegEnable(Mux(prev_fire, true.B, false.B), sx_valid_can_go) 307 sx_in(i) := RegEnable(sx_in(i-1), prev_fire) 308 } 309 } 310 val sx_last_valid = sx_valid.takeRight(1).head 311 val sx_last_ready = sx_ready.takeRight(1).head 312 val sx_last_in = sx_in.takeRight(1).head 313 sx_last_ready := !sx_last_valid || sx_last_in.uop.robIdx.needFlush(io.redirect) || io.stout.ready 314 315 io.stout.valid := sx_last_valid && !sx_last_in.uop.robIdx.needFlush(io.redirect) 316 io.stout.bits := sx_last_in 317 318 io.debug_ls := DontCare 319 io.debug_ls.s1.isTlbFirstMiss := io.tlb.resp.valid && io.tlb.resp.bits.miss && io.tlb.resp.bits.debug.isFirstIssue 320 io.debug_ls.s1_robIdx := s1_in.uop.robIdx.value 321 322 private def printPipeLine(pipeline: LsPipelineBundle, cond: Bool, name: String): Unit = { 323 XSDebug(cond, 324 p"$name" + p" pc ${Hexadecimal(pipeline.uop.cf.pc)} " + 325 p"addr ${Hexadecimal(pipeline.vaddr)} -> ${Hexadecimal(pipeline.paddr)} " + 326 p"op ${Binary(pipeline.uop.ctrl.fuOpType)} " + 327 p"data ${Hexadecimal(pipeline.data)} " + 328 p"mask ${Hexadecimal(pipeline.mask)}\n" 329 ) 330 } 331 332 printPipeLine(s0_out, s0_valid, "S0") 333 printPipeLine(s1_out, s1_valid, "S1") 334 335 // perf cnt 336 XSPerfAccumulate("s0_in_valid", s0_valid) 337 XSPerfAccumulate("s0_in_fire", s0_fire) 338 XSPerfAccumulate("s0_in_fire_first_issue", s0_fire && s0_isFirstIssue) 339 XSPerfAccumulate("s0_addr_spec_success", s0_fire && s0_saddr(VAddrBits-1, 12) === s0_in.src(0)(VAddrBits-1, 12)) 340 XSPerfAccumulate("s0_addr_spec_failed", s0_fire && s0_saddr(VAddrBits-1, 12) =/= s0_in.src(0)(VAddrBits-1, 12)) 341 XSPerfAccumulate("s0_addr_spec_success_once", s0_fire && s0_saddr(VAddrBits-1, 12) === s0_in.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 342 XSPerfAccumulate("s0_addr_spec_failed_once", s0_fire && s0_saddr(VAddrBits-1, 12) =/= s0_in.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 343 344 XSPerfAccumulate("s1_in_valid", s1_valid) 345 XSPerfAccumulate("s1_in_fire", s1_fire) 346 XSPerfAccumulate("s1_in_fire_first_issue", s1_fire && s1_in.isFirstIssue) 347 XSPerfAccumulate("s1_tlb_miss", s1_fire && s1_tlb_miss) 348 XSPerfAccumulate("s1_tlb_miss_first_issue", s1_fire && s1_tlb_miss && s1_in.isFirstIssue) 349 // end 350}