1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import xiangshan._ 23import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 24import utils._ 25import utility._ 26import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 27import freechips.rocketchip.tilelink._ 28 29class PTWReapterIO(Width: Int)(implicit p: Parameters) extends MMUIOBaseBundle { 30 val tlb = Flipped(new TlbPtwIO(Width)) 31 val ptw = new TlbPtwIO 32 33 def apply(tlb: TlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 34 this.tlb <> tlb 35 this.ptw <> ptw 36 this.sfence <> sfence 37 this.csr <> csr 38 } 39 40 def apply(tlb: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 41 this.tlb <> tlb 42 this.sfence <> sfence 43 this.csr <> csr 44 } 45 46} 47 48class PTWRepeater(Width: Int = 1, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 49 val io = IO(new PTWReapterIO(Width)) 50 51 val req_in = if (Width == 1) { 52 io.tlb.req(0) 53 } else { 54 val arb = Module(new RRArbiter(io.tlb.req(0).bits.cloneType, Width)) 55 arb.io.in <> io.tlb.req 56 arb.io.out 57 } 58 val (tlb, ptw, flush) = (io.tlb, io.ptw, DelayN(io.sfence.valid || io.csr.satp.changed || io.csr.vsatp.changed || io.csr.hgatp.changed, FenceDelay)) 59 val req = RegEnable(req_in.bits, req_in.fire) 60 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire) 61 val haveOne = BoolStopWatch(req_in.fire, tlb.resp.fire || flush) 62 val sent = BoolStopWatch(ptw.req(0).fire, req_in.fire || flush) 63 val recv = BoolStopWatch(ptw.resp.fire && haveOne, req_in.fire || flush) 64 65 req_in.ready := !haveOne 66 ptw.req(0).valid := haveOne && !sent 67 ptw.req(0).bits := req 68 69 tlb.resp.bits := resp 70 tlb.resp.valid := haveOne && recv 71 ptw.resp.ready := !recv 72 73 XSPerfAccumulate("req_count", ptw.req(0).fire) 74 XSPerfAccumulate("tlb_req_cycle", BoolStopWatch(req_in.fire, tlb.resp.fire || flush)) 75 XSPerfAccumulate("ptw_req_cycle", BoolStopWatch(ptw.req(0).fire, ptw.resp.fire || flush)) 76 77 XSDebug(haveOne, p"haveOne:${haveOne} sent:${sent} recv:${recv} sfence:${flush} req:${req} resp:${resp}") 78 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 79 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 80 assert(!RegNext(recv && io.ptw.resp.valid, init = false.B), "re-receive ptw.resp") 81 XSError(io.ptw.req(0).valid && io.ptw.resp.valid && !flush, "ptw repeater recv resp when sending") 82 XSError(io.ptw.resp.valid && (req.vpn =/= io.ptw.resp.bits.s1.entry.tag), "ptw repeater recv resp with wrong tag") 83 XSError(io.ptw.resp.valid && !io.ptw.resp.ready, "ptw repeater's ptw resp back, but not ready") 84 TimeOutAssert(sent && !recv, timeOutThreshold, "Repeater doesn't recv resp in time") 85} 86 87/* dtlb 88 * 89 */ 90 91class PTWRepeaterNB(Width: Int = 1, passReady: Boolean = false, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 92 val io = IO(new PTWReapterIO(Width)) 93 94 val req_in = if (Width == 1) { 95 io.tlb.req(0) 96 } else { 97 val arb = Module(new RRArbiter(io.tlb.req(0).bits.cloneType, Width)) 98 arb.io.in <> io.tlb.req 99 arb.io.out 100 } 101 val (tlb, ptw, flush) = (io.tlb, io.ptw, DelayN(io.sfence.valid || io.csr.satp.changed || (io.csr.priv.virt && io.csr.vsatp.changed), FenceDelay)) 102 /* sent: tlb -> repeater -> ptw 103 * recv: ptw -> repeater -> tlb 104 * different from PTWRepeater 105 */ 106 107 // tlb -> repeater -> ptw 108 val req = RegEnable(req_in.bits, req_in.fire) 109 val sent = BoolStopWatch(req_in.fire, ptw.req(0).fire || flush) 110 req_in.ready := !sent || { if (passReady) ptw.req(0).ready else false.B } 111 ptw.req(0).valid := sent 112 ptw.req(0).bits := req 113 114 // ptw -> repeater -> tlb 115 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire) 116 val recv = BoolStopWatch(ptw.resp.fire, tlb.resp.fire || flush) 117 ptw.resp.ready := !recv || { if (passReady) tlb.resp.ready else false.B } 118 tlb.resp.valid := recv 119 tlb.resp.bits := resp 120 121 XSPerfAccumulate("req", req_in.fire) 122 XSPerfAccumulate("resp", tlb.resp.fire) 123 if (!passReady) { 124 XSPerfAccumulate("req_blank", req_in.valid && sent && ptw.req(0).ready) 125 XSPerfAccumulate("resp_blank", ptw.resp.valid && recv && tlb.resp.ready) 126 XSPerfAccumulate("req_blank_ignore_ready", req_in.valid && sent) 127 XSPerfAccumulate("resp_blank_ignore_ready", ptw.resp.valid && recv) 128 } 129 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 130 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 131} 132 133class PTWFilterIO(Width: Int, hasHint: Boolean = false)(implicit p: Parameters) extends MMUIOBaseBundle { 134 val tlb = Flipped(new VectorTlbPtwIO(Width)) 135 val ptw = new TlbPtwIO() 136 val hint = if (hasHint) Some(new TlbHintIO) else None 137 val rob_head_miss_in_tlb = Output(Bool()) 138 val debugTopDown = new Bundle { 139 val robHeadVaddr = Flipped(Valid(UInt(VAddrBits.W))) 140 } 141 142 def apply(tlb: VectorTlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 143 this.tlb <> tlb 144 this.ptw <> ptw 145 this.sfence <> sfence 146 this.csr <> csr 147 } 148 149 def apply(tlb: VectorTlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 150 this.tlb <> tlb 151 this.sfence <> sfence 152 this.csr <> csr 153 } 154 155} 156 157class PTWFilterEntryIO(Width: Int, hasHint: Boolean = false)(implicit p: Parameters) extends PTWFilterIO(Width, hasHint){ 158 val flush = Input(Bool()) 159 val refill = Output(Bool()) 160 val getGpa = Output(Bool()) 161 val memidx = Output(new MemBlockidxBundle) 162} 163 164class PTWFilterEntry(Width: Int, Size: Int, hasHint: Boolean = false)(implicit p: Parameters) extends XSModule with HasPtwConst { 165 166 val io = IO(new PTWFilterEntryIO(Width, hasHint)) 167 require(isPow2(Size), s"Filter Size ($Size) must be a power of 2") 168 169 def firstValidIndex(v: Seq[Bool], valid: Bool): UInt = { 170 val index = WireInit(0.U(log2Up(Size).W)) 171 for (i <- 0 until v.size) { 172 when (v(i) === valid) { 173 index := i.U 174 } 175 } 176 index 177 } 178 179 val v = RegInit(VecInit(Seq.fill(Size)(false.B))) 180 val sent = RegInit(VecInit(Seq.fill(Size)(false.B))) 181 val vpn = Reg(Vec(Size, UInt(vpnLen.W))) 182 val s2xlate = Reg(Vec(Size, UInt(2.W))) 183 val getGpa = Reg(Vec(Size, Bool())) 184 val memidx = Reg(Vec(Size, new MemBlockidxBundle)) 185 186 val enqvalid = WireInit(VecInit(Seq.fill(Width)(false.B))) 187 val canenq = WireInit(VecInit(Seq.fill(Width)(false.B))) 188 val enqidx = WireInit(VecInit(Seq.fill(Width)(0.U(log2Up(Size).W)))) 189 190 //val selectCount = RegInit(0.U(log2Up(Width).W)) 191 192 val entryIsMatchVec = WireInit(VecInit(Seq.fill(Width)(false.B))) 193 val entryMatchIndexVec = WireInit(VecInit(Seq.fill(Width)(0.U(log2Up(Size).W)))) 194 val ptwResp_EntryMatchVec = vpn.zip(v).zip(s2xlate).map{ case ((pi, vi), s2xlatei) => vi && s2xlatei === io.ptw.resp.bits.s2xlate && io.ptw.resp.bits.hit(pi, io.csr.satp.asid, io.csr.vsatp.asid, io.csr.hgatp.asid, true, true)} 195 val ptwResp_EntryMatchFirst = firstValidIndex(ptwResp_EntryMatchVec, true.B) 196 val ptwResp_ReqMatchVec = io.tlb.req.map(a => io.ptw.resp.valid && a.bits.s2xlate === io.ptw.resp.bits.s2xlate && io.ptw.resp.bits.hit(a.bits.vpn, 0.U, 0.U, io.csr.hgatp.asid, allType = true, true)) 197 198 io.refill := Cat(ptwResp_EntryMatchVec).orR && io.ptw.resp.fire 199 io.ptw.resp.ready := true.B 200 // DontCare 201 io.tlb.req.map(_.ready := true.B) 202 io.tlb.resp.valid := false.B 203 io.tlb.resp.bits.data := 0.U.asTypeOf(new PtwRespS2withMemIdx) 204 io.tlb.resp.bits.vector := 0.U.asTypeOf(Vec(Width, Bool())) 205 io.tlb.resp.bits.getGpa := 0.U.asTypeOf(Vec(Width, Bool())) 206 io.memidx := 0.U.asTypeOf(new MemBlockidxBundle) 207 io.getGpa := 0.U 208 209 // ugly code, should be optimized later 210 require(Width <= 3, s"DTLB Filter Width ($Width) must equal or less than 3") 211 if (Width == 1) { 212 require(Size == 8, s"prefetch filter Size ($Size) should be 8") 213 canenq(0) := !(Cat(v).andR) 214 enqidx(0) := firstValidIndex(v, false.B) 215 } else if (Width == 2) { 216 require(Size == 8, s"store filter Size ($Size) should be 8") 217 canenq(0) := !(Cat(v.take(Size/2)).andR) 218 enqidx(0) := firstValidIndex(v.take(Size/2), false.B) 219 canenq(1) := !(Cat(v.drop(Size/2)).andR) 220 enqidx(1) := firstValidIndex(v.drop(Size/2), false.B) + (Size/2).U 221 } else if (Width == 3) { 222 require(Size == 16, s"load filter Size ($Size) should be 16") 223 canenq(0) := !(Cat(v.take(8)).andR) 224 enqidx(0) := firstValidIndex(v.take(8), false.B) 225 canenq(1) := !(Cat(v.drop(8).take(4)).andR) 226 enqidx(1) := firstValidIndex(v.drop(8).take(4), false.B) + 8.U 227 // four entries for prefetch 228 canenq(2) := !(Cat(v.drop(12)).andR) 229 enqidx(2) := firstValidIndex(v.drop(12), false.B) + 12.U 230 } 231 232 for (i <- 0 until Width) { 233 enqvalid(i) := io.tlb.req(i).valid && !ptwResp_ReqMatchVec(i) && !entryIsMatchVec(i) && canenq(i) 234 when (!enqvalid(i)) { 235 enqidx(i) := entryMatchIndexVec(i) 236 } 237 238 val entryIsMatch = vpn.zip(v).zip(s2xlate).map{ case ((pi, vi), s2xlatei) => vi && s2xlatei === io.tlb.req(i).bits.s2xlate && pi === io.tlb.req(i).bits.vpn} 239 entryIsMatchVec(i) := Cat(entryIsMatch).orR 240 entryMatchIndexVec(i) := firstValidIndex(entryIsMatch, true.B) 241 242 if (i > 0) { 243 for (j <- 0 until i) { 244 val newIsMatch = io.tlb.req(i).bits.vpn === io.tlb.req(j).bits.vpn && io.tlb.req(i).bits.s2xlate === io.tlb.req(j).bits.s2xlate 245 when (newIsMatch && io.tlb.req(j).valid) { 246 enqidx(i) := enqidx(j) 247 canenq(i) := canenq(j) 248 enqvalid(i) := false.B 249 } 250 } 251 } 252 253 when (enqvalid(i)) { 254 v(enqidx(i)) := true.B 255 sent(enqidx(i)) := false.B 256 vpn(enqidx(i)) := io.tlb.req(i).bits.vpn 257 s2xlate(enqidx(i)) := io.tlb.req(i).bits.s2xlate 258 getGpa(enqidx(i)) := io.tlb.req(i).bits.getGpa 259 memidx(enqidx(i)) := io.tlb.req(i).bits.memidx 260 } 261 } 262 263 val issuevec = v.zip(sent).map{ case (v, s) => v && !s} 264 val issueindex = firstValidIndex(issuevec, true.B) 265 val canissue = Cat(issuevec).orR 266 for (i <- 0 until Size) { 267 io.ptw.req(0).valid := canissue 268 io.ptw.req(0).bits.vpn := vpn(issueindex) 269 io.ptw.req(0).bits.s2xlate := s2xlate(issueindex) 270 } 271 when (io.ptw.req(0).fire) { 272 sent(issueindex) := true.B 273 } 274 275 when (io.ptw.resp.fire) { 276 v.zip(ptwResp_EntryMatchVec).map{ case (vi, mi) => when (mi) { vi := false.B }} 277 io.memidx := memidx(ptwResp_EntryMatchFirst) 278 io.getGpa := getGpa(ptwResp_EntryMatchFirst) 279 } 280 281 when (io.flush) { 282 v.map(_ := false.B) 283 } 284 285 if (hasHint) { 286 val hintIO = io.hint.getOrElse(new TlbHintIO) 287 for (i <- 0 until exuParameters.LduCnt) { 288 hintIO.req(i).id := enqidx(i) 289 hintIO.req(i).full := !canenq(i) || ptwResp_ReqMatchVec(i) 290 } 291 hintIO.resp.valid := io.refill 292 hintIO.resp.bits.id := ptwResp_EntryMatchFirst 293 hintIO.resp.bits.replay_all := PopCount(ptwResp_EntryMatchVec) > 1.U 294 } 295 296 io.rob_head_miss_in_tlb := VecInit(v.zip(vpn).map{case (vi, vpni) => { 297 vi && io.debugTopDown.robHeadVaddr.valid && vpni === get_pn(io.debugTopDown.robHeadVaddr.bits) 298 }}).asUInt.orR 299 300 301 // Perf Counter 302 val counter = PopCount(v) 303 val inflight_counter = RegInit(0.U(log2Up(Size).W)) 304 val inflight_full = inflight_counter === Size.U 305 when (io.ptw.req(0).fire =/= io.ptw.resp.fire) { 306 inflight_counter := Mux(io.ptw.req(0).fire, inflight_counter + 1.U, inflight_counter - 1.U) 307 } 308 309 assert(inflight_counter <= Size.U, "inflight should be no more than Size") 310 when (counter === 0.U) { 311 assert(!io.ptw.req(0).fire, "when counter is 0, should not req") 312 } 313 314 when (io.flush) { 315 inflight_counter := 0.U 316 } 317 318 XSPerfAccumulate("tlb_req_count", PopCount(Cat(io.tlb.req.map(_.valid)))) 319 XSPerfAccumulate("tlb_req_count_filtered", PopCount(enqvalid)) 320 XSPerfAccumulate("ptw_req_count", io.ptw.req(0).fire) 321 XSPerfAccumulate("ptw_req_cycle", inflight_counter) 322 XSPerfAccumulate("tlb_resp_count", io.tlb.resp.fire) 323 XSPerfAccumulate("ptw_resp_count", io.ptw.resp.fire) 324 XSPerfAccumulate("inflight_cycle", Cat(sent).orR) 325 326 for (i <- 0 until Size + 1) { 327 XSPerfAccumulate(s"counter${i}", counter === i.U) 328 } 329 330 for (i <- 0 until Size) { 331 TimeOutAssert(v(i), timeOutThreshold, s"Filter ${i} doesn't recv resp in time") 332 } 333 334} 335 336class PTWNewFilter(Width: Int, Size: Int, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 337 require(Size >= Width) 338 339 val io = IO(new PTWFilterIO(Width, hasHint = true)) 340 341 val load_filter = VecInit(Seq.fill(1) { 342 val load_entry = Module(new PTWFilterEntry(Width = exuParameters.LduCnt + 1, Size = loadfiltersize, hasHint = true)) 343 load_entry.io 344 }) 345 346 val store_filter = VecInit(Seq.fill(1) { 347 val store_entry = Module(new PTWFilterEntry(Width = exuParameters.StuCnt, Size = storefiltersize)) 348 store_entry.io 349 }) 350 351 val prefetch_filter = VecInit(Seq.fill(1) { 352 val prefetch_entry = Module(new PTWFilterEntry(Width = 1, Size = prefetchfiltersize)) 353 prefetch_entry.io 354 }) 355 356 val filter = load_filter ++ store_filter ++ prefetch_filter 357 358 load_filter.map(_.tlb.req := io.tlb.req.take(exuParameters.LduCnt + 1)) 359 store_filter.map(_.tlb.req := io.tlb.req.drop(exuParameters.LduCnt + 1).take(exuParameters.StuCnt)) 360 prefetch_filter.map(_.tlb.req := io.tlb.req.drop(exuParameters.LduCnt + 1 + exuParameters.StuCnt)) 361 362 val flush = DelayN(io.sfence.valid || io.csr.satp.changed || (io.csr.priv.virt && io.csr.vsatp.changed), FenceDelay) 363 val ptwResp = RegEnable(io.ptw.resp.bits, io.ptw.resp.fire) 364 val ptwResp_valid = Cat(filter.map(_.refill)).orR 365 filter.map(_.tlb.resp.ready := true.B) 366 filter.map(_.ptw.resp.valid := RegNext(io.ptw.resp.fire, init = false.B)) 367 filter.map(_.ptw.resp.bits := ptwResp) 368 filter.map(_.flush := flush) 369 filter.map(_.sfence := io.sfence) 370 filter.map(_.csr := io.csr) 371 filter.map(_.debugTopDown.robHeadVaddr := io.debugTopDown.robHeadVaddr) 372 373 io.tlb.req.map(_.ready := true.B) 374 io.tlb.resp.valid := ptwResp_valid 375 io.tlb.resp.bits.data.s2xlate := ptwResp.s2xlate 376 io.tlb.resp.bits.data.getGpa := DontCare // not used 377 io.tlb.resp.bits.data.s1 := ptwResp.s1 378 io.tlb.resp.bits.data.s2 := ptwResp.s2 379 io.tlb.resp.bits.data.memidx := 0.U.asTypeOf(new MemBlockidxBundle) 380 // vector used to represent different requestors of DTLB 381 // (e.g. the store DTLB has StuCnt requestors) 382 // However, it is only necessary to distinguish between different DTLB now 383 for (i <- 0 until Width) { 384 io.tlb.resp.bits.vector(i) := false.B 385 io.tlb.resp.bits.getGpa(i) := false.B 386 } 387 io.tlb.resp.bits.vector(0) := load_filter(0).refill 388 io.tlb.resp.bits.vector(exuParameters.LduCnt + 1) := store_filter(0).refill 389 io.tlb.resp.bits.vector(exuParameters.LduCnt + 1 + exuParameters.StuCnt) := prefetch_filter(0).refill 390 io.tlb.resp.bits.getGpa(0) := load_filter(0).getGpa 391 io.tlb.resp.bits.getGpa(exuParameters.LduCnt + 1) := store_filter(0).getGpa 392 io.tlb.resp.bits.getGpa(exuParameters.LduCnt + 1 + exuParameters.StuCnt) := prefetch_filter(0).getGpa 393 394 val hintIO = io.hint.getOrElse(new TlbHintIO) 395 val load_hintIO = load_filter(0).hint.getOrElse(new TlbHintIO) 396 for (i <- 0 until exuParameters.LduCnt) { 397 hintIO.req(i) := RegNext(load_hintIO.req(i)) 398 } 399 hintIO.resp := RegNext(load_hintIO.resp) 400 401 when (load_filter(0).refill) { 402 io.tlb.resp.bits.vector(0) := true.B 403 io.tlb.resp.bits.data.memidx := load_filter(0).memidx 404 } 405 when (store_filter(0).refill) { 406 io.tlb.resp.bits.vector(exuParameters.LduCnt + 1) := true.B 407 io.tlb.resp.bits.data.memidx := store_filter(0).memidx 408 } 409 when (prefetch_filter(0).refill) { 410 io.tlb.resp.bits.vector(exuParameters.LduCnt + 1 + exuParameters.StuCnt) := true.B 411 io.tlb.resp.bits.data.memidx := 0.U.asTypeOf(new MemBlockidxBundle) 412 } 413 414 val ptw_arb = Module(new RRArbiterInit(new PtwReq, 3)) 415 for (i <- 0 until 3) { 416 ptw_arb.io.in(i).valid := filter(i).ptw.req(0).valid 417 ptw_arb.io.in(i).bits.vpn := filter(i).ptw.req(0).bits.vpn 418 ptw_arb.io.in(i).bits.s2xlate := filter(i).ptw.req(0).bits.s2xlate 419 filter(i).ptw.req(0).ready := ptw_arb.io.in(i).ready 420 } 421 ptw_arb.io.out.ready := io.ptw.req(0).ready 422 io.ptw.req(0).valid := ptw_arb.io.out.valid 423 io.ptw.req(0).bits.vpn := ptw_arb.io.out.bits.vpn 424 io.ptw.req(0).bits.s2xlate := ptw_arb.io.out.bits.s2xlate 425 io.ptw.resp.ready := true.B 426 427 io.rob_head_miss_in_tlb := Cat(filter.map(_.rob_head_miss_in_tlb)).orR 428} 429 430class PTWFilter(Width: Int, Size: Int, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 431 require(Size >= Width) 432 433 val io = IO(new PTWFilterIO(Width)) 434 435 val v = RegInit(VecInit(Seq.fill(Size)(false.B))) 436 val ports = Reg(Vec(Size, Vec(Width, Bool()))) // record which port(s) the entry come from, may not able to cover all the ports 437 val vpn = Reg(Vec(Size, UInt(vpnLen.W))) 438 val s2xlate = Reg(Vec(Size, UInt(2.W))) 439 val getGpa = Reg(Vec(Size, Bool())) 440 val memidx = Reg(Vec(Size, new MemBlockidxBundle)) 441 val enqPtr = RegInit(0.U(log2Up(Size).W)) // Enq 442 val issPtr = RegInit(0.U(log2Up(Size).W)) // Iss to Ptw 443 val deqPtr = RegInit(0.U(log2Up(Size).W)) // Deq 444 val mayFullDeq = RegInit(false.B) 445 val mayFullIss = RegInit(false.B) 446 val counter = RegInit(0.U(log2Up(Size+1).W)) 447 val flush = DelayN(io.sfence.valid || io.csr.satp.changed || (io.csr.priv.virt && io.csr.vsatp.changed), FenceDelay) 448 val tlb_req = WireInit(io.tlb.req) // NOTE: tlb_req is not io.tlb.req, see below codes, just use cloneType 449 tlb_req.suggestName("tlb_req") 450 451 val inflight_counter = RegInit(0.U(log2Up(Size + 1).W)) 452 val inflight_full = inflight_counter === Size.U 453 454 def ptwResp_hit(vpn: UInt, s2xlate: UInt, resp: PtwRespS2): Bool = { 455 val enableS2xlate = resp.s2xlate =/= noS2xlate 456 val onlyS2 = resp.s2xlate === onlyStage2 457 val s1hit = resp.s1.hit(vpn, 0.U, io.csr.hgatp.asid, true, true, enableS2xlate) 458 val s2hit = resp.s2.hit(vpn, io.csr.hgatp.asid) 459 s2xlate === resp.s2xlate && Mux(enableS2xlate && onlyS2, s2hit, s1hit) 460 } 461 462 when (io.ptw.req(0).fire =/= io.ptw.resp.fire) { 463 inflight_counter := Mux(io.ptw.req(0).fire, inflight_counter + 1.U, inflight_counter - 1.U) 464 } 465 466 val canEnqueue = Wire(Bool()) // NOTE: actually enqueue 467 val ptwResp = RegEnable(io.ptw.resp.bits, io.ptw.resp.fire) 468 val ptwResp_OldMatchVec = vpn.zip(v).zip(s2xlate).map { case (((vpn, v), s2xlate)) =>{ 469 v && ptwResp_hit(vpn, s2xlate, io.ptw.resp.bits) 470 } 471 } 472 val ptwResp_valid = RegNext(io.ptw.resp.fire && Cat(ptwResp_OldMatchVec).orR, init = false.B) 473 // May send repeated requests to L2 tlb with same vpn(26, 3) when sector tlb 474 val oldMatchVec_early = io.tlb.req.map(a => vpn.zip(v).zip(s2xlate).map{ case ((pi, vi), s2xlate) => vi && pi === a.bits.vpn && s2xlate === a.bits.s2xlate }) 475 val lastReqMatchVec_early = io.tlb.req.map(a => tlb_req.map{ b => b.valid && b.bits.vpn === a.bits.vpn && canEnqueue && b.bits.s2xlate === a.bits.s2xlate}) 476 val newMatchVec_early = io.tlb.req.map(a => io.tlb.req.map(b => a.bits.vpn === b.bits.vpn && a.bits.s2xlate === b.bits.s2xlate)) 477 478 (0 until Width) foreach { i => 479 tlb_req(i).valid := RegNext(io.tlb.req(i).valid && 480 !(ptwResp_valid && ptwResp_hit(io.tlb.req(i).bits.vpn, io.tlb.req(i).bits.s2xlate, ptwResp)) && 481 !Cat(lastReqMatchVec_early(i)).orR, 482 init = false.B) 483 tlb_req(i).bits := RegEnable(io.tlb.req(i).bits, io.tlb.req(i).valid) 484 } 485 486 val oldMatchVec = oldMatchVec_early.map(a => RegNext(Cat(a).orR)) 487 val newMatchVec = (0 until Width).map(i => (0 until Width).map(j => 488 RegNext(newMatchVec_early(i)(j)) && tlb_req(j).valid 489 )) 490 val ptwResp_newMatchVec = tlb_req.map(a => 491 ptwResp_valid && ptwResp_hit(a.bits.vpn, a.bits.s2xlate, ptwResp)) 492 493 val oldMatchVec2 = (0 until Width).map(i => oldMatchVec_early(i).map(RegNext(_)).map(_ & tlb_req(i).valid)) 494 val update_ports = v.indices.map(i => oldMatchVec2.map(j => j(i))) 495 val ports_init = (0 until Width).map(i => (1 << i).U(Width.W)) 496 val filter_ports = (0 until Width).map(i => ParallelMux(newMatchVec(i).zip(ports_init).drop(i))) 497 val resp_vector = RegEnable(ParallelMux(ptwResp_OldMatchVec zip ports), io.ptw.resp.fire) 498 val resp_getGpa = RegEnable(ParallelMux(ptwResp_OldMatchVec zip getGpa), io.ptw.resp.fire) 499 500 def canMerge(index: Int) : Bool = { 501 ptwResp_newMatchVec(index) || oldMatchVec(index) || 502 Cat(newMatchVec(index).take(index)).orR 503 } 504 505 def filter_req() = { 506 val reqs = tlb_req.indices.map{ i => 507 val req = Wire(ValidIO(new PtwReqwithMemIdx())) 508 val merge = canMerge(i) 509 req.bits := tlb_req(i).bits 510 req.valid := !merge && tlb_req(i).valid 511 req 512 } 513 reqs 514 } 515 516 val reqs = filter_req() 517 val req_ports = filter_ports 518 val isFull = enqPtr === deqPtr && mayFullDeq 519 val isEmptyDeq = enqPtr === deqPtr && !mayFullDeq 520 val isEmptyIss = enqPtr === issPtr && !mayFullIss 521 val accumEnqNum = (0 until Width).map(i => PopCount(reqs.take(i).map(_.valid))) 522 val enqPtrVecInit = VecInit((0 until Width).map(i => enqPtr + i.U)) 523 val enqPtrVec = VecInit((0 until Width).map(i => enqPtrVecInit(accumEnqNum(i)))) 524 val enqNum = PopCount(reqs.map(_.valid)) 525 canEnqueue := counter +& enqNum <= Size.U 526 527 // the req may recv false ready, but actually received. Filter and TLB will handle it. 528 val enqNum_fake = PopCount(io.tlb.req.map(_.valid)) 529 val canEnqueue_fake = counter +& enqNum_fake <= Size.U 530 io.tlb.req.map(_.ready := canEnqueue_fake) // NOTE: just drop un-fire reqs 531 532 // tlb req flushed by ptw resp: last ptw resp && current ptw resp 533 // the flushed tlb req will fakely enq, with a false valid 534 val tlb_req_flushed = reqs.map(a => io.ptw.resp.valid && ptwResp_hit(a.bits.vpn, a.bits.s2xlate, io.ptw.resp.bits)) 535 536 io.tlb.resp.valid := ptwResp_valid 537 io.tlb.resp.bits.data.s2xlate := ptwResp.s2xlate 538 io.tlb.resp.bits.data.s1 := ptwResp.s1 539 io.tlb.resp.bits.data.s2 := ptwResp.s2 540 io.tlb.resp.bits.data.memidx := memidx(OHToUInt(ptwResp_OldMatchVec)) 541 io.tlb.resp.bits.vector := resp_vector 542 io.tlb.resp.bits.data.getGpa := getGpa(OHToUInt(ptwResp_OldMatchVec)) 543 io.tlb.resp.bits.getGpa := DontCare 544 545 val issue_valid = v(issPtr) && !isEmptyIss && !inflight_full 546 val issue_filtered = ptwResp_valid && ptwResp_hit(io.ptw.req(0).bits.vpn, io.ptw.req(0).bits.s2xlate, ptwResp) 547 val issue_fire_fake = issue_valid && (io.ptw.req(0).ready || (issue_filtered && false.B /*timing-opt*/)) 548 io.ptw.req(0).valid := issue_valid && !issue_filtered 549 io.ptw.req(0).bits.vpn := vpn(issPtr) 550 io.ptw.req(0).bits.s2xlate := s2xlate(issPtr) 551 io.ptw.resp.ready := true.B 552 553 reqs.zipWithIndex.map{ 554 case (req, i) => 555 when (req.valid && canEnqueue) { 556 v(enqPtrVec(i)) := !tlb_req_flushed(i) 557 vpn(enqPtrVec(i)) := req.bits.vpn 558 s2xlate(enqPtrVec(i)) := req.bits.s2xlate 559 getGpa(enqPtrVec(i)) := req.bits.getGpa 560 memidx(enqPtrVec(i)) := req.bits.memidx 561 ports(enqPtrVec(i)) := req_ports(i).asBools 562 } 563 } 564 for (i <- ports.indices) { 565 when (v(i)) { 566 ports(i) := ports(i).zip(update_ports(i)).map(a => a._1 || a._2) 567 } 568 } 569 570 val do_enq = canEnqueue && Cat(reqs.map(_.valid)).orR 571 val do_deq = (!v(deqPtr) && !isEmptyDeq) 572 val do_iss = issue_fire_fake || (!v(issPtr) && !isEmptyIss) 573 when (do_enq) { 574 enqPtr := enqPtr + enqNum 575 } 576 when (do_deq) { 577 deqPtr := deqPtr + 1.U 578 } 579 when (do_iss) { 580 issPtr := issPtr + 1.U 581 } 582 when (issue_fire_fake && issue_filtered) { // issued but is filtered 583 v(issPtr) := false.B 584 } 585 when (do_enq =/= do_deq) { 586 mayFullDeq := do_enq 587 } 588 when (do_enq =/= do_iss) { 589 mayFullIss := do_enq 590 } 591 592 when (io.ptw.resp.fire) { 593 v.zip(ptwResp_OldMatchVec).map{ case (vi, mi) => when (mi) { vi := false.B }} 594 } 595 596 counter := counter - do_deq + Mux(do_enq, enqNum, 0.U) 597 assert(counter <= Size.U, "counter should be no more than Size") 598 assert(inflight_counter <= Size.U, "inflight should be no more than Size") 599 when (counter === 0.U) { 600 assert(!io.ptw.req(0).fire, "when counter is 0, should not req") 601 assert(isEmptyDeq && isEmptyIss, "when counter is 0, should be empty") 602 } 603 when (counter === Size.U) { 604 assert(mayFullDeq, "when counter is Size, should be full") 605 } 606 607 when (flush) { 608 v.map(_ := false.B) 609 deqPtr := 0.U 610 enqPtr := 0.U 611 issPtr := 0.U 612 ptwResp_valid := false.B 613 mayFullDeq := false.B 614 mayFullIss := false.B 615 counter := 0.U 616 inflight_counter := 0.U 617 } 618 619 val robHeadVaddr = io.debugTopDown.robHeadVaddr 620 io.rob_head_miss_in_tlb := VecInit(v.zip(vpn).map{case (vi, vpni) => { 621 vi && robHeadVaddr.valid && vpni === get_pn(robHeadVaddr.bits) 622 }}).asUInt.orR 623 624 // perf 625 XSPerfAccumulate("tlb_req_count", PopCount(Cat(io.tlb.req.map(_.valid)))) 626 XSPerfAccumulate("tlb_req_count_filtered", Mux(do_enq, accumEnqNum(Width - 1), 0.U)) 627 XSPerfAccumulate("ptw_req_count", io.ptw.req(0).fire) 628 XSPerfAccumulate("ptw_req_cycle", inflight_counter) 629 XSPerfAccumulate("tlb_resp_count", io.tlb.resp.fire) 630 XSPerfAccumulate("ptw_resp_count", io.ptw.resp.fire) 631 XSPerfAccumulate("inflight_cycle", !isEmptyDeq) 632 for (i <- 0 until Size + 1) { 633 XSPerfAccumulate(s"counter${i}", counter === i.U) 634 } 635 636 for (i <- 0 until Size) { 637 TimeOutAssert(v(i), timeOutThreshold, s"Filter ${i} doesn't recv resp in time") 638 } 639} 640 641object PTWRepeater { 642 def apply(fenceDelay: Int, 643 tlb: TlbPtwIO, 644 sfence: SfenceBundle, 645 csr: TlbCsrBundle 646 )(implicit p: Parameters) = { 647 val width = tlb.req.size 648 val repeater = Module(new PTWRepeater(width, fenceDelay)) 649 repeater.io.apply(tlb, sfence, csr) 650 repeater 651 } 652 653 def apply(fenceDelay: Int, 654 tlb: TlbPtwIO, 655 ptw: TlbPtwIO, 656 sfence: SfenceBundle, 657 csr: TlbCsrBundle 658 )(implicit p: Parameters) = { 659 val width = tlb.req.size 660 val repeater = Module(new PTWRepeater(width, fenceDelay)) 661 repeater.io.apply(tlb, ptw, sfence, csr) 662 repeater 663 } 664} 665 666object PTWRepeaterNB { 667 def apply(passReady: Boolean, fenceDelay: Int, 668 tlb: TlbPtwIO, 669 sfence: SfenceBundle, 670 csr: TlbCsrBundle 671 )(implicit p: Parameters) = { 672 val width = tlb.req.size 673 val repeater = Module(new PTWRepeaterNB(width, passReady,fenceDelay)) 674 repeater.io.apply(tlb, sfence, csr) 675 repeater 676 } 677 678 def apply(passReady: Boolean, fenceDelay: Int, 679 tlb: TlbPtwIO, 680 ptw: TlbPtwIO, 681 sfence: SfenceBundle, 682 csr: TlbCsrBundle 683 )(implicit p: Parameters) = { 684 val width = tlb.req.size 685 val repeater = Module(new PTWRepeaterNB(width, passReady, fenceDelay)) 686 repeater.io.apply(tlb, ptw, sfence, csr) 687 repeater 688 } 689} 690 691object PTWFilter { 692 def apply(fenceDelay: Int, 693 tlb: VectorTlbPtwIO, 694 ptw: TlbPtwIO, 695 sfence: SfenceBundle, 696 csr: TlbCsrBundle, 697 size: Int 698 )(implicit p: Parameters) = { 699 val width = tlb.req.size 700 val filter = Module(new PTWFilter(width, size, fenceDelay)) 701 filter.io.apply(tlb, ptw, sfence, csr) 702 filter 703 } 704 705 def apply(fenceDelay: Int, 706 tlb: VectorTlbPtwIO, 707 sfence: SfenceBundle, 708 csr: TlbCsrBundle, 709 size: Int 710 )(implicit p: Parameters) = { 711 val width = tlb.req.size 712 val filter = Module(new PTWFilter(width, size, fenceDelay)) 713 filter.io.apply(tlb, sfence, csr) 714 filter 715 } 716} 717 718object PTWNewFilter { 719 def apply(fenceDelay: Int, 720 tlb: VectorTlbPtwIO, 721 ptw: TlbPtwIO, 722 sfence: SfenceBundle, 723 csr: TlbCsrBundle, 724 size: Int 725 )(implicit p: Parameters) = { 726 val width = tlb.req.size 727 val filter = Module(new PTWNewFilter(width, size, fenceDelay)) 728 filter.io.apply(tlb, ptw, sfence, csr) 729 filter 730 } 731 732 def apply(fenceDelay: Int, 733 tlb: VectorTlbPtwIO, 734 sfence: SfenceBundle, 735 csr: TlbCsrBundle, 736 size: Int 737 )(implicit p: Parameters) = { 738 val width = tlb.req.size 739 val filter = Module(new PTWNewFilter(width, size, fenceDelay)) 740 filter.io.apply(tlb, sfence, csr) 741 filter 742 } 743} 744