1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import freechips.rocketchip.tilelink.ClientMetadata 23import utils.{HasPerfEvents, XSDebug, XSPerfAccumulate} 24import xiangshan.L1CacheErrorInfo 25 26class LoadPipe(id: Int)(implicit p: Parameters) extends DCacheModule with HasPerfEvents { 27 val io = IO(new DCacheBundle { 28 // incoming requests 29 val lsu = Flipped(new DCacheLoadIO) 30 // req got nacked in stage 0? 31 val nack = Input(Bool()) 32 33 // meta and data array read port 34 val meta_read = DecoupledIO(new MetaReadReq) 35 val meta_resp = Input(Vec(nWays, new Meta)) 36 val error_flag_resp = Input(Vec(nWays, Bool())) 37 38 val tag_read = DecoupledIO(new TagReadReq) 39 val tag_resp = Input(Vec(nWays, UInt(encTagBits.W))) 40 41 val banked_data_read = DecoupledIO(new L1BankedDataReadReq) 42 val banked_data_resp = Input(Vec(DCacheBanks, new L1BankedDataReadResult())) 43 val read_error_delayed = Input(Bool()) 44 45 // banked data read conflict 46 val bank_conflict_slow = Input(Bool()) 47 val bank_conflict_fast = Input(Bool()) 48 49 // send miss request to miss queue 50 val miss_req = DecoupledIO(new MissReq) 51 52 // update state vec in replacement algo 53 val replace_access = ValidIO(new ReplacementAccessBundle) 54 // find the way to be replaced 55 val replace_way = new ReplacementWayReqIO 56 57 // load fast wakeup should be disabled when data read is not ready 58 val disable_ld_fast_wakeup = Input(Bool()) 59 60 // ecc error 61 val error = Output(new L1CacheErrorInfo()) 62 }) 63 64 assert(RegNext(io.meta_read.ready)) 65 66 val s1_ready = Wire(Bool()) 67 val s2_ready = Wire(Bool()) 68 // LSU requests 69 // it you got nacked, you can directly passdown 70 val not_nacked_ready = io.meta_read.ready && io.tag_read.ready && s1_ready 71 val nacked_ready = true.B 72 73 // ready can wait for valid 74 io.lsu.req.ready := (!io.nack && not_nacked_ready) || (io.nack && nacked_ready) 75 io.meta_read.valid := io.lsu.req.fire() && !io.nack 76 io.tag_read.valid := io.lsu.req.fire() && !io.nack 77 78 val meta_read = io.meta_read.bits 79 val tag_read = io.tag_read.bits 80 81 // Tag read for new requests 82 meta_read.idx := get_idx(io.lsu.req.bits.addr) 83 meta_read.way_en := ~0.U(nWays.W) 84 // meta_read.tag := DontCare 85 86 tag_read.idx := get_idx(io.lsu.req.bits.addr) 87 tag_read.way_en := ~0.U(nWays.W) 88 89 // Pipeline 90 // -------------------------------------------------------------------------------- 91 // stage 0 92 // -------------------------------------------------------------------------------- 93 // read tag 94 95 val s0_valid = io.lsu.req.fire() 96 val s0_req = io.lsu.req.bits 97 val s0_fire = s0_valid && s1_ready 98 99 assert(RegNext(!(s0_valid && (s0_req.cmd =/= MemoryOpConstants.M_XRD && s0_req.cmd =/= MemoryOpConstants.M_PFR && s0_req.cmd =/= MemoryOpConstants.M_PFW))), "LoadPipe only accepts load req / softprefetch read or write!") 100 dump_pipeline_reqs("LoadPipe s0", s0_valid, s0_req) 101 102 // -------------------------------------------------------------------------------- 103 // stage 1 104 // -------------------------------------------------------------------------------- 105 // tag match, read data 106 107 val s1_valid = RegInit(false.B) 108 val s1_req = RegEnable(s0_req, s0_fire) 109 // in stage 1, load unit gets the physical address 110 val s1_addr = io.lsu.s1_paddr 111 val s1_vaddr = s1_req.addr 112 val s1_bank_oh = UIntToOH(addr_to_dcache_bank(s1_req.addr)) 113 val s1_nack = RegNext(io.nack) 114 val s1_nack_data = !io.banked_data_read.ready 115 val s1_fire = s1_valid && s2_ready 116 s1_ready := !s1_valid || s1_fire 117 118 when (s0_fire) { s1_valid := true.B } 119 .elsewhen (s1_fire) { s1_valid := false.B } 120 121 dump_pipeline_reqs("LoadPipe s1", s1_valid, s1_req) 122 123 // tag check 124 val meta_resp = io.meta_resp 125 val tag_resp = io.tag_resp.map(r => r(tagBits - 1, 0)) 126 def wayMap[T <: Data](f: Int => T) = VecInit((0 until nWays).map(f)) 127 val s1_tag_eq_way = wayMap((w: Int) => tag_resp(w) === (get_tag(s1_addr))).asUInt 128 val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta_resp(w).coh.isValid()).asUInt 129 val s1_tag_match = s1_tag_match_way.orR 130 assert(RegNext(!s1_valid || PopCount(s1_tag_match_way) <= 1.U), "tag should not match with more than 1 way") 131 132 val s1_fake_meta = Wire(new Meta) 133// s1_fake_meta.tag := get_tag(s1_addr) 134 s1_fake_meta.coh := ClientMetadata.onReset 135 val s1_fake_tag = get_tag(s1_addr) 136 137 // when there are no tag match, we give it a Fake Meta 138 // this simplifies our logic in s2 stage 139 val s1_hit_meta = Mux(s1_tag_match, Mux1H(s1_tag_match_way, wayMap((w: Int) => meta_resp(w))), s1_fake_meta) 140 val s1_hit_coh = s1_hit_meta.coh 141 val s1_hit_error = Mux(s1_tag_match, Mux1H(s1_tag_match_way, wayMap((w: Int) => io.error_flag_resp(w))), false.B) 142 143 io.replace_way.set.valid := RegNext(s0_fire) 144 io.replace_way.set.bits := get_idx(s1_vaddr) 145 val s1_repl_way_en = UIntToOH(io.replace_way.way) 146 val s1_repl_tag = Mux1H(s1_repl_way_en, wayMap(w => tag_resp(w))) 147 val s1_repl_coh = Mux1H(s1_repl_way_en, wayMap(w => meta_resp(w).coh)) 148 149 val s1_need_replacement = !s1_tag_match 150 val s1_way_en = Mux(s1_need_replacement, s1_repl_way_en, s1_tag_match_way) 151 val s1_coh = Mux(s1_need_replacement, s1_repl_coh, s1_hit_coh) 152 val s1_tag = Mux(s1_need_replacement, s1_repl_tag, get_tag(s1_addr)) 153 154 // data read 155 io.banked_data_read.valid := s1_fire && !s1_nack 156 io.banked_data_read.bits.addr := s1_vaddr 157 io.banked_data_read.bits.way_en := s1_tag_match_way 158 159 // get s1_will_send_miss_req in lpad_s1 160 val s1_has_permission = s1_hit_coh.onAccess(s1_req.cmd)._1 161 val s1_new_hit_coh = s1_hit_coh.onAccess(s1_req.cmd)._3 162 val s1_hit = s1_tag_match && s1_has_permission && s1_hit_coh === s1_new_hit_coh 163 val s1_will_send_miss_req = s1_valid && !s1_nack && !s1_nack_data && !s1_hit 164 165 // check ecc error 166 val s1_encTag = Mux1H(s1_tag_match_way, wayMap((w: Int) => io.tag_resp(w))) 167 val s1_flag_error = Mux(s1_need_replacement, false.B, s1_hit_error) // error reported by exist dcache error bit 168 169 // -------------------------------------------------------------------------------- 170 // stage 2 171 // -------------------------------------------------------------------------------- 172 // return data 173 174 // val s2_valid = RegEnable(next = s1_valid && !io.lsu.s1_kill, init = false.B, enable = s1_fire) 175 val s2_valid = RegInit(false.B) 176 val s2_req = RegEnable(s1_req, s1_fire) 177 val s2_addr = RegEnable(s1_addr, s1_fire) 178 val s2_vaddr = RegEnable(s1_vaddr, s1_fire) 179 val s2_bank_oh = RegEnable(s1_bank_oh, s1_fire) 180 s2_ready := true.B 181 182 val s2_fire = s2_valid 183 184 when (s1_fire) { s2_valid := !io.lsu.s1_kill } 185 .elsewhen(io.lsu.resp.fire()) { s2_valid := false.B } 186 187 dump_pipeline_reqs("LoadPipe s2", s2_valid, s2_req) 188 189 // hit, miss, nack, permission checking 190 val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_fire) 191 val s2_tag_match = RegEnable(s1_tag_match, s1_fire) 192 193 val s2_hit_meta = RegEnable(s1_hit_meta, s1_fire) 194 val s2_hit_coh = RegEnable(s1_hit_coh, s1_fire) 195 val s2_has_permission = s2_hit_coh.onAccess(s2_req.cmd)._1 196 val s2_new_hit_coh = s2_hit_coh.onAccess(s2_req.cmd)._3 197 198 val s2_way_en = RegEnable(s1_way_en, s1_fire) 199 val s2_repl_coh = RegEnable(s1_repl_coh, s1_fire) 200 val s2_repl_tag = RegEnable(s1_repl_tag, s1_fire) 201 val s2_encTag = RegEnable(s1_encTag, s1_fire) 202 203 // when req got nacked, upper levels should replay this request 204 // nacked or not 205 val s2_nack_hit = RegEnable(s1_nack, s1_fire) 206 // can no allocate mshr for load miss 207 val s2_nack_no_mshr = io.miss_req.valid && !io.miss_req.ready 208 // Bank conflict on data arrays 209 val s2_nack_data = RegEnable(!io.banked_data_read.ready, s1_fire) 210 val s2_nack = s2_nack_hit || s2_nack_no_mshr || s2_nack_data 211 212 val banked_data_resp = io.banked_data_resp 213 val s2_bank_addr = addr_to_dcache_bank(s2_addr) 214 val banked_data_resp_word = Mux1H(s2_bank_oh, io.banked_data_resp) // io.banked_data_resp(s2_bank_addr) 215 dontTouch(s2_bank_addr) 216 217 val s2_instrtype = s2_req.instrtype 218 219 val s2_tag_error = dcacheParameters.tagCode.decode(s2_encTag).error // error reported by tag ecc check 220 val s2_flag_error = RegEnable(s1_flag_error, s1_fire) 221 222 val s2_hit = s2_tag_match && s2_has_permission && s2_hit_coh === s2_new_hit_coh 223 224 // only dump these signals when they are actually valid 225 dump_pipeline_valids("LoadPipe s2", "s2_hit", s2_valid && s2_hit) 226 dump_pipeline_valids("LoadPipe s2", "s2_nack", s2_valid && s2_nack) 227 dump_pipeline_valids("LoadPipe s2", "s2_nack_hit", s2_valid && s2_nack_hit) 228 dump_pipeline_valids("LoadPipe s2", "s2_nack_no_mshr", s2_valid && s2_nack_no_mshr) 229 230 val s2_can_send_miss_req = RegEnable(s1_will_send_miss_req, s1_fire) 231 232 // send load miss to miss queue 233 io.miss_req.valid := s2_valid && s2_can_send_miss_req 234 io.miss_req.bits := DontCare 235 io.miss_req.bits.source := s2_instrtype 236 io.miss_req.bits.cmd := s2_req.cmd 237 io.miss_req.bits.addr := get_block_addr(s2_addr) 238 io.miss_req.bits.vaddr := s2_vaddr 239 io.miss_req.bits.way_en := s2_way_en 240 io.miss_req.bits.req_coh := s2_hit_coh 241 io.miss_req.bits.replace_coh := s2_repl_coh 242 io.miss_req.bits.replace_tag := s2_repl_tag 243 io.miss_req.bits.cancel := io.lsu.s2_kill || s2_tag_error 244 245 // send back response 246 val resp = Wire(ValidIO(new DCacheWordResp)) 247 resp.valid := s2_valid 248 resp.bits := DontCare 249 // resp.bits.data := s2_word_decoded 250 resp.bits.data := banked_data_resp_word.raw_data 251 // * on miss or nack, upper level should replay request 252 // but if we successfully sent the request to miss queue 253 // upper level does not need to replay request 254 // they can sit in load queue and wait for refill 255 // 256 // * report a miss if bank conflict is detected 257 val real_miss = !s2_hit 258 resp.bits.miss := real_miss || io.bank_conflict_slow 259 // load pipe need replay when there is a bank conflict 260 resp.bits.replay := resp.bits.miss && (!io.miss_req.fire() || s2_nack) || io.bank_conflict_slow 261 resp.bits.tag_error := s2_tag_error // report tag_error in load s2 262 263 XSPerfAccumulate("dcache_read_bank_conflict", io.bank_conflict_slow && s2_valid) 264 265 io.lsu.resp.valid := resp.valid 266 io.lsu.resp.bits := resp.bits 267 assert(RegNext(!(resp.valid && !io.lsu.resp.ready)), "lsu should be ready in s2") 268 269 when (resp.valid) { 270 resp.bits.dump() 271 } 272 273 io.lsu.s1_hit_way := s1_tag_match_way 274 io.lsu.s1_disable_fast_wakeup := io.disable_ld_fast_wakeup 275 io.lsu.s1_bank_conflict := io.bank_conflict_fast 276 assert(RegNext(s1_ready && s2_ready), "load pipeline should never be blocked") 277 278 // -------------------------------------------------------------------------------- 279 // stage 3 280 // -------------------------------------------------------------------------------- 281 // report ecc error 282 283 val s3_valid = RegNext(s2_valid) 284 val s3_addr = RegEnable(s2_addr, s2_fire) 285 val s3_hit = RegEnable(s2_hit, s2_fire) 286 287 val s3_data_error = io.read_error_delayed // banked_data_resp_word.error && !bank_conflict 288 val s3_tag_error = RegEnable(s2_tag_error, s2_fire) 289 val s3_flag_error = RegEnable(s2_flag_error, s2_fire) 290 val s3_error = s3_tag_error || s3_flag_error || s3_data_error 291 292 // error_delayed signal will be used to update uop.exception 1 cycle after load writeback 293 resp.bits.error_delayed := s3_error && (s3_hit || s3_tag_error) 294 295 // report tag / data / l2 error (with paddr) to bus error unit 296 io.error := 0.U.asTypeOf(new L1CacheErrorInfo()) 297 io.error.report_to_beu := (s3_tag_error || s3_data_error) && s3_valid 298 io.error.paddr := s3_addr 299 io.error.source.tag := s3_tag_error 300 io.error.source.data := s3_data_error 301 io.error.source.l2 := s3_flag_error 302 io.error.opType.load := true.B 303 // report tag error / l2 corrupted to CACHE_ERROR csr 304 io.error.valid := s3_error && s3_valid 305 306 // update plru, report error in s3 307 308 io.replace_access.valid := RegNext(RegNext(RegNext(io.meta_read.fire()) && s1_valid) && !s2_nack_no_mshr) 309 io.replace_access.bits.set := RegNext(RegNext(get_idx(s1_req.addr))) 310 io.replace_access.bits.way := RegNext(RegNext(Mux(s1_tag_match, OHToUInt(s1_tag_match_way), io.replace_way.way))) 311 312 // -------------------------------------------------------------------------------- 313 // Debug logging functions 314 def dump_pipeline_reqs(pipeline_stage_name: String, valid: Bool, 315 req: DCacheWordReq ) = { 316 when (valid) { 317 XSDebug(s"$pipeline_stage_name: ") 318 req.dump() 319 } 320 } 321 322 def dump_pipeline_valids(pipeline_stage_name: String, signal_name: String, valid: Bool) = { 323 when (valid) { 324 XSDebug(s"$pipeline_stage_name $signal_name\n") 325 } 326 } 327 328 // performance counters 329 XSPerfAccumulate("load_req", io.lsu.req.fire()) 330 XSPerfAccumulate("load_s1_kill", s1_fire && io.lsu.s1_kill) 331 XSPerfAccumulate("load_hit_way", s1_fire && s1_tag_match) 332 XSPerfAccumulate("load_replay", io.lsu.resp.fire() && resp.bits.replay) 333 XSPerfAccumulate("load_replay_for_data_nack", io.lsu.resp.fire() && resp.bits.replay && s2_nack_data) 334 XSPerfAccumulate("load_replay_for_no_mshr", io.lsu.resp.fire() && resp.bits.replay && s2_nack_no_mshr) 335 XSPerfAccumulate("load_replay_for_conflict", io.lsu.resp.fire() && resp.bits.replay && io.bank_conflict_slow) 336 XSPerfAccumulate("load_hit", io.lsu.resp.fire() && !real_miss) 337 XSPerfAccumulate("load_miss", io.lsu.resp.fire() && real_miss) 338 XSPerfAccumulate("load_succeed", io.lsu.resp.fire() && !resp.bits.miss && !resp.bits.replay) 339 XSPerfAccumulate("load_miss_or_conflict", io.lsu.resp.fire() && resp.bits.miss) 340 XSPerfAccumulate("actual_ld_fast_wakeup", s1_fire && s1_tag_match && !io.disable_ld_fast_wakeup) 341 XSPerfAccumulate("ideal_ld_fast_wakeup", io.banked_data_read.fire() && s1_tag_match) 342 343 val perfEvents = Seq( 344 ("load_req ", io.lsu.req.fire() ), 345 ("load_replay ", io.lsu.resp.fire() && resp.bits.replay ), 346 ("load_replay_for_data_nack", io.lsu.resp.fire() && resp.bits.replay && s2_nack_data ), 347 ("load_replay_for_no_mshr ", io.lsu.resp.fire() && resp.bits.replay && s2_nack_no_mshr ), 348 ("load_replay_for_conflict ", io.lsu.resp.fire() && resp.bits.replay && io.bank_conflict_slow ), 349 ) 350 generatePerfEvent() 351} 352