1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.Bundles.{MemExuInput, MemExuOutput} 27import xiangshan.backend.fu.PMPRespBundle 28import xiangshan.backend.fu.FuConfig._ 29import xiangshan.backend.fu.FuType._ 30import xiangshan.backend.ctrlblock.DebugLsInfoBundle 31import xiangshan.backend.fu.NewCSR._ 32import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp} 33import xiangshan.cache.{DcacheStoreRequestIO, DCacheStoreIO, MemoryOpConstants, HasDCacheParameters, StorePrefetchReq} 34 35class StoreUnit(implicit p: Parameters) extends XSModule 36 with HasDCacheParameters 37 with HasVLSUParameters 38 { 39 val io = IO(new Bundle() { 40 val redirect = Flipped(ValidIO(new Redirect)) 41 val stin = Flipped(Decoupled(new MemExuInput)) 42 val issue = Valid(new MemExuInput) 43 val tlb = new TlbRequestIO() 44 val dcache = new DCacheStoreIO 45 val pmp = Flipped(new PMPRespBundle()) 46 val lsq = ValidIO(new LsPipelineBundle) 47 val lsq_replenish = Output(new LsPipelineBundle()) 48 val feedback_slow = ValidIO(new RSFeedback) 49 val prefetch_req = Flipped(DecoupledIO(new StorePrefetchReq)) 50 // provide prefetch info to sms 51 val prefetch_train = ValidIO(new StPrefetchTrainBundle()) 52 // speculative for gated control 53 val s1_prefetch_spec = Output(Bool()) 54 val s2_prefetch_spec = Output(Bool()) 55 val stld_nuke_query = Valid(new StoreNukeQueryIO) 56 val stout = DecoupledIO(new MemExuOutput) // writeback store 57 val vecstout = DecoupledIO(new VecPipelineFeedbackIO(isVStore = true)) 58 // store mask, send to sq in store_s0 59 val st_mask_out = Valid(new StoreMaskBundle) 60 val debug_ls = Output(new DebugLsInfoBundle) 61 // vector 62 val vecstin = Flipped(Decoupled(new VecPipeBundle(isVStore = true))) 63 val vec_isFirstIssue = Input(Bool()) 64 // trigger 65 val fromCsrTrigger = Input(new CsrTriggerBundle) 66 }) 67 68 val s1_ready, s2_ready, s3_ready = WireInit(false.B) 69 70 // Pipeline 71 // -------------------------------------------------------------------------------- 72 // stage 0 73 // -------------------------------------------------------------------------------- 74 // generate addr, use addr to query DCache and DTLB 75 val s0_iss_valid = io.stin.valid 76 val s0_prf_valid = io.prefetch_req.valid && io.dcache.req.ready 77 val s0_vec_valid = io.vecstin.valid 78 val s0_valid = s0_iss_valid || s0_prf_valid || s0_vec_valid 79 val s0_use_flow_vec = s0_vec_valid 80 val s0_use_flow_rs = s0_iss_valid && !s0_vec_valid 81 val s0_use_flow_prf = !s0_iss_valid && !s0_vec_valid && s0_prf_valid 82 val s0_stin = Mux(s0_use_flow_rs, io.stin.bits, 0.U.asTypeOf(io.stin.bits)) 83 val s0_vecstin = Mux(s0_use_flow_vec, io.vecstin.bits, 0.U.asTypeOf(io.vecstin.bits)) 84 val s0_uop = Mux(s0_use_flow_rs, s0_stin.uop, s0_vecstin.uop) 85 val s0_isFirstIssue = s0_use_flow_rs && io.stin.bits.isFirstIssue || s0_use_flow_vec && io.vec_isFirstIssue 86 val s0_size = Mux(s0_use_flow_rs || s0_use_flow_vec, s0_uop.fuOpType(2,0), 0.U)// may broken if use it in feature 87 val s0_mem_idx = Mux(s0_use_flow_rs || s0_use_flow_vec, s0_uop.sqIdx.value, 0.U) 88 val s0_rob_idx = Mux(s0_use_flow_rs || s0_use_flow_vec, s0_uop.robIdx, 0.U.asTypeOf(s0_uop.robIdx)) 89 val s0_pc = Mux(s0_use_flow_rs || s0_use_flow_vec, s0_uop.pc, 0.U) 90 val s0_instr_type = Mux(s0_use_flow_rs || s0_use_flow_vec, STORE_SOURCE.U, DCACHE_PREFETCH_SOURCE.U) 91 val s0_wlineflag = Mux(s0_use_flow_rs, s0_uop.fuOpType === LSUOpType.cbo_zero, false.B) 92 val s0_out = Wire(new LsPipelineBundle) 93 val s0_kill = s0_uop.robIdx.needFlush(io.redirect) 94 val s0_can_go = s1_ready 95 val s0_fire = s0_valid && !s0_kill && s0_can_go 96 val s0_is128bit = is128Bit(s0_vecstin.alignedType) 97 // vector 98 val s0_vecActive = !s0_use_flow_vec || s0_vecstin.vecActive 99 // val s0_flowPtr = s0_vecstin.flowPtr 100 // val s0_isLastElem = s0_vecstin.isLastElem 101 val s0_secondInv = s0_vecstin.usSecondInv 102 val s0_elemIdx = s0_vecstin.elemIdx 103 val s0_alignedType = s0_vecstin.alignedType 104 val s0_mBIndex = s0_vecstin.mBIndex 105 106 // generate addr 107 // val saddr = s0_in.bits.src(0) + SignExt(s0_in.bits.uop.imm(11,0), VAddrBits) 108 val imm12 = WireInit(s0_uop.imm(11,0)) 109 val saddr_lo = s0_stin.src(0)(11,0) + Cat(0.U(1.W), imm12) 110 val saddr_hi = Mux(saddr_lo(12), 111 Mux(imm12(11), s0_stin.src(0)(VAddrBits-1, 12), s0_stin.src(0)(VAddrBits-1, 12)+1.U), 112 Mux(imm12(11), s0_stin.src(0)(VAddrBits-1, 12)+SignExt(1.U, VAddrBits-12), s0_stin.src(0)(VAddrBits-1, 12)), 113 ) 114 val s0_saddr = Cat(saddr_hi, saddr_lo(11,0)) 115 val s0_vaddr = Mux( 116 s0_use_flow_rs, 117 s0_saddr, 118 Mux( 119 s0_use_flow_vec, 120 s0_vecstin.vaddr, 121 io.prefetch_req.bits.vaddr 122 ) 123 ) 124 val s0_mask = Mux( 125 s0_use_flow_rs, 126 genVWmask128(s0_saddr, s0_uop.fuOpType(2,0)), 127 Mux( 128 s0_use_flow_vec, 129 s0_vecstin.mask, 130 // -1.asSInt.asUInt 131 Fill(VLEN/8, 1.U(1.W)) 132 ) 133 ) 134 135 io.tlb.req.valid := s0_valid 136 io.tlb.req.bits.vaddr := s0_vaddr 137 io.tlb.req.bits.cmd := TlbCmd.write 138 io.tlb.req.bits.size := s0_size 139 io.tlb.req.bits.kill := false.B 140 io.tlb.req.bits.memidx.is_ld := false.B 141 io.tlb.req.bits.memidx.is_st := true.B 142 io.tlb.req.bits.memidx.idx := s0_mem_idx 143 io.tlb.req.bits.debug.robIdx := s0_rob_idx 144 io.tlb.req.bits.no_translate := false.B 145 io.tlb.req.bits.debug.pc := s0_pc 146 io.tlb.req.bits.debug.isFirstIssue := s0_isFirstIssue 147 io.tlb.req_kill := false.B 148 io.tlb.req.bits.hyperinst := LSUOpType.isHsv(s0_uop.fuOpType) 149 io.tlb.req.bits.hlvx := false.B 150 151 // Dcache access here: not **real** dcache write 152 // just read meta and tag in dcache, to find out the store will hit or miss 153 154 // NOTE: The store request does not wait for the dcache to be ready. 155 // If the dcache is not ready at this time, the dcache is not queried. 156 // But, store prefetch request will always wait for dcache to be ready to make progress. 157 io.dcache.req.valid := s0_fire 158 io.dcache.req.bits.cmd := MemoryOpConstants.M_PFW 159 io.dcache.req.bits.vaddr := s0_vaddr 160 io.dcache.req.bits.instrtype := s0_instr_type 161 162 s0_out := DontCare 163 s0_out.vaddr := s0_vaddr 164 // Now data use its own io 165 // s1_out.data := genWdata(s1_in.src(1), s1_in.uop.fuOpType(1,0)) 166 s0_out.data := s0_stin.src(1) 167 s0_out.uop := s0_uop 168 s0_out.miss := false.B 169 s0_out.mask := s0_mask 170 s0_out.isFirstIssue := s0_isFirstIssue 171 s0_out.isHWPrefetch := s0_use_flow_prf 172 s0_out.wlineflag := s0_wlineflag 173 s0_out.isvec := s0_use_flow_vec 174 s0_out.is128bit := s0_is128bit 175 s0_out.vecActive := s0_vecActive 176 s0_out.usSecondInv := s0_secondInv 177 s0_out.elemIdx := s0_elemIdx 178 s0_out.alignedType := s0_alignedType 179 s0_out.mbIndex := s0_mBIndex 180 when(s0_valid && s0_isFirstIssue) { 181 s0_out.uop.debugInfo.tlbFirstReqTime := GTimer() 182 } 183 184 // exception check 185 val s0_addr_aligned = LookupTree(Mux(s0_use_flow_vec, s0_vecstin.alignedType(1,0), s0_uop.fuOpType(1, 0)), List( 186 "b00".U -> true.B, //b 187 "b01".U -> (s0_out.vaddr(0) === 0.U), //h 188 "b10".U -> (s0_out.vaddr(1,0) === 0.U), //w 189 "b11".U -> (s0_out.vaddr(2,0) === 0.U) //d 190 )) 191 // if vector store sends 128-bit requests, its address must be 128-aligned 192 XSError(s0_use_flow_vec && s0_out.vaddr(3, 0) =/= 0.U && s0_vecstin.alignedType(2), "unit stride 128 bit element is not aligned!") 193 s0_out.uop.exceptionVec(storeAddrMisaligned) := Mux(s0_use_flow_rs || s0_use_flow_vec, !s0_addr_aligned, false.B) 194 195 io.st_mask_out.valid := s0_use_flow_rs || s0_use_flow_vec 196 io.st_mask_out.bits.mask := s0_out.mask 197 io.st_mask_out.bits.sqIdx := s0_out.uop.sqIdx 198 199 io.stin.ready := s1_ready && s0_use_flow_rs 200 io.vecstin.ready := s1_ready && s0_use_flow_vec 201 io.prefetch_req.ready := s1_ready && io.dcache.req.ready && !s0_iss_valid && !s0_vec_valid 202 203 // Pipeline 204 // -------------------------------------------------------------------------------- 205 // stage 1 206 // -------------------------------------------------------------------------------- 207 // TLB resp (send paddr to dcache) 208 val s1_valid = RegInit(false.B) 209 val s1_in = RegEnable(s0_out, s0_fire) 210 val s1_out = Wire(new LsPipelineBundle) 211 val s1_kill = Wire(Bool()) 212 val s1_can_go = s2_ready 213 val s1_fire = s1_valid && !s1_kill && s1_can_go 214 val s1_vecActive = RegEnable(s0_out.vecActive, true.B, s0_fire) 215 216 // mmio cbo decoder 217 val s1_mmio_cbo = s1_in.uop.fuOpType === LSUOpType.cbo_clean || 218 s1_in.uop.fuOpType === LSUOpType.cbo_flush || 219 s1_in.uop.fuOpType === LSUOpType.cbo_inval 220 val s1_paddr = io.tlb.resp.bits.paddr(0) 221 val s1_gpaddr = io.tlb.resp.bits.gpaddr(0) 222 val s1_tlb_miss = io.tlb.resp.bits.miss 223 val s1_mmio = s1_mmio_cbo 224 val s1_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, StaCfg).asUInt.orR 225 val s1_isvec = RegEnable(s0_out.isvec, false.B, s0_fire) 226 // val s1_isLastElem = RegEnable(s0_isLastElem, false.B, s0_fire) 227 s1_kill := s1_in.uop.robIdx.needFlush(io.redirect) || (s1_tlb_miss && !s1_isvec) 228 229 s1_ready := !s1_valid || s1_kill || s2_ready 230 io.tlb.resp.ready := true.B // TODO: why dtlbResp needs a ready? 231 when (s0_fire) { s1_valid := true.B } 232 .elsewhen (s1_fire) { s1_valid := false.B } 233 .elsewhen (s1_kill) { s1_valid := false.B } 234 235 // st-ld violation dectect request. 236 io.stld_nuke_query.valid := s1_valid && !s1_tlb_miss && !s1_in.isHWPrefetch 237 io.stld_nuke_query.bits.robIdx := s1_in.uop.robIdx 238 io.stld_nuke_query.bits.paddr := s1_paddr 239 io.stld_nuke_query.bits.mask := s1_in.mask 240 io.stld_nuke_query.bits.matchLine := s1_in.isvec && s1_in.is128bit 241 242 // issue 243 io.issue.valid := s1_valid && !s1_tlb_miss && !s1_in.isHWPrefetch && !s1_isvec 244 io.issue.bits := RegEnable(s0_stin, s0_valid) 245 246 247 // Send TLB feedback to store issue queue 248 // Store feedback is generated in store_s1, sent to RS in store_s2 249 val s1_feedback = Wire(Valid(new RSFeedback)) 250 s1_feedback.valid := s1_valid & !s1_in.isHWPrefetch 251 s1_feedback.bits.hit := !s1_tlb_miss 252 s1_feedback.bits.flushState := io.tlb.resp.bits.ptwBack 253 s1_feedback.bits.robIdx := s1_out.uop.robIdx 254 s1_feedback.bits.sourceType := RSFeedbackType.tlbMiss 255 s1_feedback.bits.dataInvalidSqIdx := DontCare 256 s1_feedback.bits.sqIdx := s1_out.uop.sqIdx 257 s1_feedback.bits.lqIdx := s1_out.uop.lqIdx 258 259 XSDebug(s1_feedback.valid, 260 "S1 Store: tlbHit: %d robIdx: %d\n", 261 s1_feedback.bits.hit, 262 s1_feedback.bits.robIdx.value 263 ) 264 265 // io.feedback_slow := s1_feedback 266 267 // get paddr from dtlb, check if rollback is needed 268 // writeback store inst to lsq 269 s1_out := s1_in 270 s1_out.paddr := s1_paddr 271 s1_out.gpaddr := s1_gpaddr 272 s1_out.miss := false.B 273 s1_out.mmio := s1_mmio 274 s1_out.tlbMiss := s1_tlb_miss 275 s1_out.atomic := s1_mmio 276 s1_out.uop.exceptionVec(storePageFault) := io.tlb.resp.bits.excp(0).pf.st && s1_vecActive 277 s1_out.uop.exceptionVec(storeAccessFault) := io.tlb.resp.bits.excp(0).af.st && s1_vecActive 278 s1_out.uop.exceptionVec(storeGuestPageFault) := io.tlb.resp.bits.excp(0).gpf.st && s1_vecActive 279 280 // trigger 281 val storeTrigger = Module(new StoreTrigger) 282 storeTrigger.io.fromCsrTrigger.tdataVec := io.fromCsrTrigger.tdataVec 283 storeTrigger.io.fromCsrTrigger.tEnableVec := io.fromCsrTrigger.tEnableVec 284 storeTrigger.io.fromCsrTrigger.triggerCanRaiseBpExp := io.fromCsrTrigger.triggerCanRaiseBpExp 285 storeTrigger.io.fromStore.vaddr := s1_in.vaddr 286 287 s1_out.uop.trigger.backendHit := storeTrigger.io.toStore.triggerHitVec 288 s1_out.uop.trigger.backendCanFire := storeTrigger.io.toStore.triggerCanFireVec 289 s1_out.uop.exceptionVec(breakPoint) := storeTrigger.io.toStore.breakPointExp 290 291 // scalar store and scalar load nuke check, and also other purposes 292 io.lsq.valid := s1_valid && !s1_in.isHWPrefetch 293 io.lsq.bits := s1_out 294 io.lsq.bits.miss := s1_tlb_miss 295 296 // kill dcache write intent request when tlb miss or exception 297 io.dcache.s1_kill := (s1_tlb_miss || s1_exception || s1_mmio || s1_in.uop.robIdx.needFlush(io.redirect)) 298 io.dcache.s1_paddr := s1_paddr 299 300 // write below io.out.bits assign sentence to prevent overwriting values 301 val s1_tlb_memidx = io.tlb.resp.bits.memidx 302 when(s1_tlb_memidx.is_st && io.tlb.resp.valid && !s1_tlb_miss && s1_tlb_memidx.idx === s1_out.uop.sqIdx.value) { 303 // printf("Store idx = %d\n", s1_tlb_memidx.idx) 304 s1_out.uop.debugInfo.tlbRespTime := GTimer() 305 } 306 307 // Pipeline 308 // -------------------------------------------------------------------------------- 309 // stage 2 310 // -------------------------------------------------------------------------------- 311 // mmio check 312 val s2_valid = RegInit(false.B) 313 val s2_in = RegEnable(s1_out, s1_fire) 314 val s2_out = Wire(new LsPipelineBundle) 315 val s2_kill = Wire(Bool()) 316 val s2_can_go = s3_ready 317 val s2_fire = s2_valid && !s2_kill && s2_can_go 318 val s2_vecActive = RegEnable(s1_out.vecActive, true.B, s1_fire) 319 320 s2_ready := !s2_valid || s2_kill || s3_ready 321 when (s1_fire) { s2_valid := true.B } 322 .elsewhen (s2_fire) { s2_valid := false.B } 323 .elsewhen (s2_kill) { s2_valid := false.B } 324 325 val s2_pmp = WireInit(io.pmp) 326 327 val s2_exception = (ExceptionNO.selectByFu(s2_out.uop.exceptionVec, StaCfg).asUInt.orR) && RegNext(s1_feedback.bits.hit) 328 val s2_mmio = (s2_in.mmio || s2_pmp.mmio) && RegNext(s1_feedback.bits.hit) 329 s2_kill := ((s2_mmio && !s2_exception) && !s2_in.isvec) || s2_in.uop.robIdx.needFlush(io.redirect) 330 331 s2_out := s2_in 332 s2_out.af := s2_pmp.st && !s2_in.isvec 333 s2_out.mmio := s2_mmio && !s2_exception 334 s2_out.atomic := s2_in.atomic || s2_pmp.atomic 335 s2_out.uop.exceptionVec(storeAccessFault) := (s2_in.uop.exceptionVec(storeAccessFault) || 336 s2_pmp.st || 337 (s2_in.isvec && s2_pmp.mmio && RegNext(s1_feedback.bits.hit)) 338 ) && s2_vecActive 339 340 // kill dcache write intent request when mmio or exception 341 io.dcache.s2_kill := (s2_mmio || s2_exception || s2_in.uop.robIdx.needFlush(io.redirect)) 342 io.dcache.s2_pc := s2_out.uop.pc 343 // TODO: dcache resp 344 io.dcache.resp.ready := true.B 345 346 // feedback tlb miss to RS in store_s2 347 val feedback_slow_valid = WireInit(false.B) 348 feedback_slow_valid := s1_feedback.valid && !s1_out.uop.robIdx.needFlush(io.redirect) && !s1_out.isvec 349 io.feedback_slow.valid := GatedValidRegNext(feedback_slow_valid) 350 io.feedback_slow.bits := RegEnable(s1_feedback.bits, feedback_slow_valid) 351 352 val s2_vecFeedback = RegNext(!s1_out.uop.robIdx.needFlush(io.redirect) && s1_feedback.bits.hit) && s2_in.isvec 353 354 // mmio and exception 355 io.lsq_replenish := s2_out 356 io.lsq_replenish.af := s2_out.af && !s2_kill 357 358 // prefetch related 359 io.lsq_replenish.miss := io.dcache.resp.fire && io.dcache.resp.bits.miss // miss info 360 361 // RegNext prefetch train for better timing 362 // ** Now, prefetch train is valid at store s3 ** 363 val s2_prefetch_train_valid = WireInit(false.B) 364 s2_prefetch_train_valid := s2_valid && io.dcache.resp.fire && !s2_out.mmio && !s2_in.tlbMiss && !s2_in.isHWPrefetch 365 if(EnableStorePrefetchSMS) { 366 io.s1_prefetch_spec := s1_fire 367 io.s2_prefetch_spec := s2_prefetch_train_valid 368 io.prefetch_train.valid := RegNext(s2_prefetch_train_valid) 369 io.prefetch_train.bits.fromLsPipelineBundle(s2_in, latch = true, enable = s2_prefetch_train_valid) 370 }else { 371 io.s1_prefetch_spec := false.B 372 io.s2_prefetch_spec := false.B 373 io.prefetch_train.valid := false.B 374 io.prefetch_train.bits.fromLsPipelineBundle(s2_in, latch = true, enable = false.B) 375 } 376 // override miss bit 377 io.prefetch_train.bits.miss := RegEnable(io.dcache.resp.bits.miss, s2_prefetch_train_valid) 378 // TODO: add prefetch and access bit 379 io.prefetch_train.bits.meta_prefetch := false.B 380 io.prefetch_train.bits.meta_access := false.B 381 382 // Pipeline 383 // -------------------------------------------------------------------------------- 384 // stage 3 385 // -------------------------------------------------------------------------------- 386 // store write back 387 val s3_valid = RegInit(false.B) 388 val s3_in = RegEnable(s2_out, s2_fire) 389 val s3_out = Wire(new MemExuOutput(isVector = true)) 390 val s3_kill = s3_in.uop.robIdx.needFlush(io.redirect) 391 val s3_can_go = s3_ready 392 val s3_fire = s3_valid && !s3_kill && s3_can_go 393 val s3_vecFeedback = RegEnable(s2_vecFeedback, s2_fire) 394 395 when (s2_fire) { s3_valid := (!s2_mmio || s2_exception) && !s2_out.isHWPrefetch } 396 .elsewhen (s3_fire) { s3_valid := false.B } 397 .elsewhen (s3_kill) { s3_valid := false.B } 398 399 // wb: writeback 400 val SelectGroupSize = RollbackGroupSize 401 val lgSelectGroupSize = log2Ceil(SelectGroupSize) 402 val TotalSelectCycles = scala.math.ceil(log2Ceil(LoadQueueRAWSize).toFloat / lgSelectGroupSize).toInt + 1 403 404 s3_out := DontCare 405 s3_out.uop := s3_in.uop 406 s3_out.data := DontCare 407 s3_out.debug.isMMIO := s3_in.mmio 408 s3_out.debug.paddr := s3_in.paddr 409 s3_out.debug.vaddr := s3_in.vaddr 410 s3_out.debug.isPerfCnt := false.B 411 412 // Pipeline 413 // -------------------------------------------------------------------------------- 414 // stage x 415 // -------------------------------------------------------------------------------- 416 // delay TotalSelectCycles - 2 cycle(s) 417 val TotalDelayCycles = TotalSelectCycles - 2 418 val sx_valid = Wire(Vec(TotalDelayCycles + 1, Bool())) 419 val sx_ready = Wire(Vec(TotalDelayCycles + 1, Bool())) 420 val sx_in = Wire(Vec(TotalDelayCycles + 1, new VecMemExuOutput(isVector = true))) 421 422 // backward ready signal 423 s3_ready := sx_ready.head 424 for (i <- 0 until TotalDelayCycles + 1) { 425 if (i == 0) { 426 sx_valid(i) := s3_valid 427 sx_in(i).output := s3_out 428 sx_in(i).vecFeedback := s3_vecFeedback 429 sx_in(i).mmio := s3_in.mmio 430 sx_in(i).usSecondInv := s3_in.usSecondInv 431 sx_in(i).elemIdx := s3_in.elemIdx 432 sx_in(i).alignedType := s3_in.alignedType 433 sx_in(i).mbIndex := s3_in.mbIndex 434 sx_in(i).mask := s3_in.mask 435 sx_in(i).vaddr := s3_in.vaddr 436 sx_ready(i) := !s3_valid(i) || sx_in(i).output.uop.robIdx.needFlush(io.redirect) || (if (TotalDelayCycles == 0) io.stout.ready else sx_ready(i+1)) 437 } else { 438 val cur_kill = sx_in(i).output.uop.robIdx.needFlush(io.redirect) 439 val cur_can_go = (if (i == TotalDelayCycles) io.stout.ready else sx_ready(i+1)) 440 val cur_fire = sx_valid(i) && !cur_kill && cur_can_go 441 val prev_fire = sx_valid(i-1) && !sx_in(i-1).output.uop.robIdx.needFlush(io.redirect) && sx_ready(i) 442 443 sx_ready(i) := !sx_valid(i) || cur_kill || (if (i == TotalDelayCycles) io.stout.ready else sx_ready(i+1)) 444 val sx_valid_can_go = prev_fire || cur_fire || cur_kill 445 sx_valid(i) := RegEnable(Mux(prev_fire, true.B, false.B), false.B, sx_valid_can_go) 446 sx_in(i) := RegEnable(sx_in(i-1), prev_fire) 447 } 448 } 449 val sx_last_valid = sx_valid.takeRight(1).head 450 val sx_last_ready = sx_ready.takeRight(1).head 451 val sx_last_in = sx_in.takeRight(1).head 452 sx_last_ready := !sx_last_valid || sx_last_in.output.uop.robIdx.needFlush(io.redirect) || io.stout.ready 453 454 io.stout.valid := sx_last_valid && !sx_last_in.output.uop.robIdx.needFlush(io.redirect) && isStore(sx_last_in.output.uop.fuType) 455 io.stout.bits := sx_last_in.output 456 io.stout.bits.uop.exceptionVec := ExceptionNO.selectByFu(sx_last_in.output.uop.exceptionVec, StaCfg) 457 458 io.vecstout.valid := sx_last_valid && !sx_last_in.output.uop.robIdx.needFlush(io.redirect) && isVStore(sx_last_in.output.uop.fuType) 459 // TODO: implement it! 460 io.vecstout.bits.mBIndex := sx_last_in.mbIndex 461 io.vecstout.bits.hit := sx_last_in.vecFeedback 462 io.vecstout.bits.isvec := true.B 463 io.vecstout.bits.sourceType := RSFeedbackType.tlbMiss 464 io.vecstout.bits.flushState := DontCare 465 io.vecstout.bits.mmio := sx_last_in.mmio 466 io.vecstout.bits.exceptionVec := ExceptionNO.selectByFu(sx_last_in.output.uop.exceptionVec, VstuCfg) 467 io.vecstout.bits.usSecondInv := sx_last_in.usSecondInv 468 io.vecstout.bits.vecFeedback := sx_last_in.vecFeedback 469 io.vecstout.bits.elemIdx := sx_last_in.elemIdx 470 io.vecstout.bits.alignedType := sx_last_in.alignedType 471 io.vecstout.bits.mask := sx_last_in.mask 472 io.vecstout.bits.vaddr := sx_last_in.vaddr 473 // io.vecstout.bits.reg_offset.map(_ := DontCare) 474 // io.vecstout.bits.elemIdx.map(_ := sx_last_in.elemIdx) 475 // io.vecstout.bits.elemIdxInsideVd.map(_ := DontCare) 476 // io.vecstout.bits.vecdata.map(_ := DontCare) 477 // io.vecstout.bits.mask.map(_ := DontCare) 478 // io.vecstout.bits.alignedType.map(_ := sx_last_in.alignedType) 479 480 io.debug_ls := DontCare 481 io.debug_ls.s1_robIdx := s1_in.uop.robIdx.value 482 io.debug_ls.s1_isTlbFirstMiss := io.tlb.resp.valid && io.tlb.resp.bits.miss && io.tlb.resp.bits.debug.isFirstIssue && !s1_in.isHWPrefetch 483 484 private def printPipeLine(pipeline: LsPipelineBundle, cond: Bool, name: String): Unit = { 485 XSDebug(cond, 486 p"$name" + p" pc ${Hexadecimal(pipeline.uop.pc)} " + 487 p"addr ${Hexadecimal(pipeline.vaddr)} -> ${Hexadecimal(pipeline.paddr)} " + 488 p"op ${Binary(pipeline.uop.fuOpType)} " + 489 p"data ${Hexadecimal(pipeline.data)} " + 490 p"mask ${Hexadecimal(pipeline.mask)}\n" 491 ) 492 } 493 494 printPipeLine(s0_out, s0_valid, "S0") 495 printPipeLine(s1_out, s1_valid, "S1") 496 497 // perf cnt 498 XSPerfAccumulate("s0_in_valid", s0_valid) 499 XSPerfAccumulate("s0_in_fire", s0_fire) 500 XSPerfAccumulate("s0_vecin_fire", s0_fire && s0_use_flow_vec) 501 XSPerfAccumulate("s0_in_fire_first_issue", s0_fire && s0_isFirstIssue) 502 XSPerfAccumulate("s0_addr_spec_success", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) === s0_stin.src(0)(VAddrBits-1, 12)) 503 XSPerfAccumulate("s0_addr_spec_failed", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) =/= s0_stin.src(0)(VAddrBits-1, 12)) 504 XSPerfAccumulate("s0_addr_spec_success_once", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) === s0_stin.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 505 XSPerfAccumulate("s0_addr_spec_failed_once", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) =/= s0_stin.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 506 507 XSPerfAccumulate("s1_in_valid", s1_valid) 508 XSPerfAccumulate("s1_in_fire", s1_fire) 509 XSPerfAccumulate("s1_in_fire_first_issue", s1_fire && s1_in.isFirstIssue) 510 XSPerfAccumulate("s1_tlb_miss", s1_fire && s1_tlb_miss) 511 XSPerfAccumulate("s1_tlb_miss_first_issue", s1_fire && s1_tlb_miss && s1_in.isFirstIssue) 512 // end 513}