1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import xiangshan._ 23import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 24import utils._ 25import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 26import freechips.rocketchip.tilelink._ 27 28class PTWReapterIO(Width: Int)(implicit p: Parameters) extends MMUIOBaseBundle { 29 val tlb = Flipped(new TlbPtwIO(Width)) 30 val ptw = new TlbPtwIO 31 32 def apply(tlb: TlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 33 this.tlb <> tlb 34 this.ptw <> ptw 35 this.sfence <> sfence 36 this.csr <> csr 37 } 38 39 def apply(tlb: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 40 this.tlb <> tlb 41 this.sfence <> sfence 42 this.csr <> csr 43 } 44 45} 46 47class PTWRepeater(Width: Int = 1, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 48 val io = IO(new PTWReapterIO(Width)) 49 50 val req_in = if (Width == 1) { 51 io.tlb.req(0) 52 } else { 53 val arb = Module(new RRArbiter(io.tlb.req(0).bits.cloneType, Width)) 54 arb.io.in <> io.tlb.req 55 arb.io.out 56 } 57 val (tlb, ptw, flush) = (io.tlb, io.ptw, DelayN(io.sfence.valid || io.csr.satp.changed, FenceDelay)) 58 val req = RegEnable(req_in.bits, req_in.fire()) 59 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire()) 60 val haveOne = BoolStopWatch(req_in.fire(), tlb.resp.fire() || flush) 61 val sent = BoolStopWatch(ptw.req(0).fire(), req_in.fire() || flush) 62 val recv = BoolStopWatch(ptw.resp.fire(), req_in.fire() || flush) 63 64 req_in.ready := !haveOne 65 ptw.req(0).valid := haveOne && !sent 66 ptw.req(0).bits := req 67 68 tlb.resp.bits := resp 69 tlb.resp.valid := haveOne && recv 70 ptw.resp.ready := !recv 71 72 XSPerfAccumulate("req_count", ptw.req(0).fire()) 73 XSPerfAccumulate("tlb_req_cycle", BoolStopWatch(req_in.fire(), tlb.resp.fire() || flush)) 74 XSPerfAccumulate("ptw_req_cycle", BoolStopWatch(ptw.req(0).fire(), ptw.resp.fire() || flush)) 75 76 XSDebug(haveOne, p"haveOne:${haveOne} sent:${sent} recv:${recv} sfence:${flush} req:${req} resp:${resp}") 77 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 78 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 79 assert(!RegNext(recv && io.ptw.resp.valid, init = false.B), "re-receive ptw.resp") 80 TimeOutAssert(sent && !recv, timeOutThreshold, "Repeater doesn't recv resp in time") 81} 82 83/* dtlb 84 * 85 */ 86 87class PTWRepeaterNB(Width: Int = 1, passReady: Boolean = false, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 88 val io = IO(new PTWReapterIO(Width)) 89 90 val req_in = if (Width == 1) { 91 io.tlb.req(0) 92 } else { 93 val arb = Module(new RRArbiter(io.tlb.req(0).bits.cloneType, Width)) 94 arb.io.in <> io.tlb.req 95 arb.io.out 96 } 97 val (tlb, ptw, flush) = (io.tlb, io.ptw, DelayN(io.sfence.valid || io.csr.satp.changed, FenceDelay)) 98 /* sent: tlb -> repeater -> ptw 99 * recv: ptw -> repeater -> tlb 100 * different from PTWRepeater 101 */ 102 103 // tlb -> repeater -> ptw 104 val req = RegEnable(req_in.bits, req_in.fire()) 105 val sent = BoolStopWatch(req_in.fire(), ptw.req(0).fire() || flush) 106 req_in.ready := !sent || { if (passReady) ptw.req(0).ready else false.B } 107 ptw.req(0).valid := sent 108 ptw.req(0).bits := req 109 110 // ptw -> repeater -> tlb 111 val resp = RegEnable(ptw.resp.bits, ptw.resp.fire()) 112 val recv = BoolStopWatch(ptw.resp.fire(), tlb.resp.fire() || flush) 113 ptw.resp.ready := !recv || { if (passReady) tlb.resp.ready else false.B } 114 tlb.resp.valid := recv 115 tlb.resp.bits := resp 116 117 XSPerfAccumulate("req", req_in.fire()) 118 XSPerfAccumulate("resp", tlb.resp.fire()) 119 if (!passReady) { 120 XSPerfAccumulate("req_blank", req_in.valid && sent && ptw.req(0).ready) 121 XSPerfAccumulate("resp_blank", ptw.resp.valid && recv && tlb.resp.ready) 122 XSPerfAccumulate("req_blank_ignore_ready", req_in.valid && sent) 123 XSPerfAccumulate("resp_blank_ignore_ready", ptw.resp.valid && recv) 124 } 125 XSDebug(req_in.valid || io.tlb.resp.valid, p"tlb: ${tlb}\n") 126 XSDebug(io.ptw.req(0).valid || io.ptw.resp.valid, p"ptw: ${ptw}\n") 127} 128 129class PTWFilterIO(Width: Int)(implicit p: Parameters) extends MMUIOBaseBundle { 130 val tlb = Flipped(new VectorTlbPtwIO(Width)) 131 val ptw = new TlbPtwIO() 132 133 def apply(tlb: VectorTlbPtwIO, ptw: TlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 134 this.tlb <> tlb 135 this.ptw <> ptw 136 this.sfence <> sfence 137 this.csr <> csr 138 } 139 140 def apply(tlb: VectorTlbPtwIO, sfence: SfenceBundle, csr: TlbCsrBundle): Unit = { 141 this.tlb <> tlb 142 this.sfence <> sfence 143 this.csr <> csr 144 } 145 146} 147 148class PTWFilter(Width: Int, Size: Int, FenceDelay: Int)(implicit p: Parameters) extends XSModule with HasPtwConst { 149 require(Size >= Width) 150 151 val io = IO(new PTWFilterIO(Width)) 152 153 val v = RegInit(VecInit(Seq.fill(Size)(false.B))) 154 val ports = Reg(Vec(Size, Vec(Width, Bool()))) // record which port(s) the entry come from, may not able to cover all the ports 155 val vpn = Reg(Vec(Size, UInt(vpnLen.W))) 156 val enqPtr = RegInit(0.U(log2Up(Size).W)) // Enq 157 val issPtr = RegInit(0.U(log2Up(Size).W)) // Iss to Ptw 158 val deqPtr = RegInit(0.U(log2Up(Size).W)) // Deq 159 val mayFullDeq = RegInit(false.B) 160 val mayFullIss = RegInit(false.B) 161 val counter = RegInit(0.U(log2Up(Size+1).W)) 162 163 val flush = DelayN(io.sfence.valid || io.csr.satp.changed, FenceDelay) 164 val tlb_req = WireInit(io.tlb.req) // NOTE: tlb_req is not io.tlb.req, see below codes, just use cloneType 165 tlb_req.suggestName("tlb_req") 166 167 val inflight_counter = RegInit(0.U(log2Up(Size + 1).W)) 168 val inflight_full = inflight_counter === Size.U 169 when (io.ptw.req(0).fire() =/= io.ptw.resp.fire()) { 170 inflight_counter := Mux(io.ptw.req(0).fire(), inflight_counter + 1.U, inflight_counter - 1.U) 171 } 172 173 val ptwResp = RegEnable(io.ptw.resp.bits, io.ptw.resp.fire()) 174 val ptwResp_OldMatchVec = vpn.zip(v).map{ case (pi, vi) => 175 vi && io.ptw.resp.bits.entry.hit(pi, io.csr.satp.asid, true, true)} 176 val ptwResp_valid = RegNext(io.ptw.resp.fire() && Cat(ptwResp_OldMatchVec).orR, init = false.B) 177 val oldMatchVec_early = io.tlb.req.map(a => vpn.zip(v).map{ case (pi, vi) => vi && pi === a.bits.vpn}) 178 val lastReqMatchVec_early = io.tlb.req.map(a => tlb_req.map{ b => b.valid && b.bits.vpn === a.bits.vpn}) 179 val newMatchVec_early = io.tlb.req.map(a => io.tlb.req.map(b => a.bits.vpn === b.bits.vpn)) 180 181 (0 until Width) foreach { i => 182 tlb_req(i).valid := RegNext(io.tlb.req(i).valid && 183 !(ptwResp_valid && ptwResp.entry.hit(io.tlb.req(i).bits.vpn, 0.U, true, true)) && 184 !Cat(lastReqMatchVec_early(i)).orR, 185 init = false.B) 186 tlb_req(i).bits := RegEnable(io.tlb.req(i).bits, io.tlb.req(i).valid) 187 } 188 189 val oldMatchVec = oldMatchVec_early.map(a => RegNext(Cat(a).orR)) 190 val newMatchVec = (0 until Width).map(i => (0 until Width).map(j => 191 RegNext(newMatchVec_early(i)(j)) && tlb_req(j).valid 192 )) 193 val ptwResp_newMatchVec = tlb_req.map(a => 194 ptwResp_valid && ptwResp.entry.hit(a.bits.vpn, 0.U, allType = true, true)) 195 196 val oldMatchVec2 = (0 until Width).map(i => oldMatchVec_early(i).map(RegNext(_)).map(_ & tlb_req(i).valid)) 197 val update_ports = v.indices.map(i => oldMatchVec2.map(j => j(i))) 198 val ports_init = (0 until Width).map(i => (1 << i).U(Width.W)) 199 val filter_ports = (0 until Width).map(i => ParallelMux(newMatchVec(i).zip(ports_init).drop(i))) 200 val resp_vector = RegEnable(ParallelMux(ptwResp_OldMatchVec zip ports), io.ptw.resp.fire()) 201 202 def canMerge(index: Int) : Bool = { 203 ptwResp_newMatchVec(index) || oldMatchVec(index) || 204 Cat(newMatchVec(index).take(index)).orR 205 } 206 207 def filter_req() = { 208 val reqs = tlb_req.indices.map{ i => 209 val req = Wire(ValidIO(new PtwReq())) 210 val merge = canMerge(i) 211 req.bits := tlb_req(i).bits 212 req.valid := !merge && tlb_req(i).valid 213 req 214 } 215 reqs 216 } 217 218 val reqs = filter_req() 219 val req_ports = filter_ports 220 val isFull = enqPtr === deqPtr && mayFullDeq 221 val isEmptyDeq = enqPtr === deqPtr && !mayFullDeq 222 val isEmptyIss = enqPtr === issPtr && !mayFullIss 223 val accumEnqNum = (0 until Width).map(i => PopCount(reqs.take(i).map(_.valid))) 224 val enqPtrVecInit = VecInit((0 until Width).map(i => enqPtr + i.U)) 225 val enqPtrVec = VecInit((0 until Width).map(i => enqPtrVecInit(accumEnqNum(i)))) 226 val enqNum = PopCount(reqs.map(_.valid)) 227 val canEnqueue = counter +& enqNum <= Size.U 228 229 // the req may recv false ready, but actually received. Filter and TLB will handle it. 230 val enqNum_fake = PopCount(io.tlb.req.map(_.valid)) 231 val canEnqueue_fake = counter +& enqNum_fake <= Size.U 232 io.tlb.req.map(_.ready := canEnqueue_fake) // NOTE: just drop un-fire reqs 233 234 io.tlb.resp.valid := ptwResp_valid 235 io.tlb.resp.bits.data := ptwResp 236 io.tlb.resp.bits.vector := resp_vector 237 238 val issue_valid = v(issPtr) && !isEmptyIss && !inflight_full 239 val issue_filtered = ptwResp_valid && ptwResp.entry.hit(io.ptw.req(0).bits.vpn, io.csr.satp.asid, allType=true, ignoreAsid=true) 240 val issue_fire_fake = issue_valid && (io.ptw.req(0).ready || (issue_filtered && false.B /*timing-opt*/)) 241 io.ptw.req(0).valid := issue_valid && !issue_filtered 242 io.ptw.req(0).bits.vpn := vpn(issPtr) 243 io.ptw.resp.ready := true.B 244 245 reqs.zipWithIndex.map{ 246 case (req, i) => 247 when (req.valid && canEnqueue) { 248 v(enqPtrVec(i)) := true.B 249 vpn(enqPtrVec(i)) := req.bits.vpn 250 ports(enqPtrVec(i)) := req_ports(i).asBools 251 } 252 } 253 for (i <- ports.indices) { 254 when (v(i)) { 255 ports(i) := ports(i).zip(update_ports(i)).map(a => a._1 || a._2) 256 } 257 } 258 259 val do_enq = canEnqueue && Cat(reqs.map(_.valid)).orR 260 val do_deq = (!v(deqPtr) && !isEmptyDeq) 261 val do_iss = issue_fire_fake || (!v(issPtr) && !isEmptyIss) 262 when (do_enq) { 263 enqPtr := enqPtr + enqNum 264 } 265 when (do_deq) { 266 deqPtr := deqPtr + 1.U 267 } 268 when (do_iss) { 269 issPtr := issPtr + 1.U 270 } 271 when (issue_fire_fake && issue_filtered) { // issued but is filtered 272 v(issPtr) := false.B 273 } 274 when (do_enq =/= do_deq) { 275 mayFullDeq := do_enq 276 } 277 when (do_enq =/= do_iss) { 278 mayFullIss := do_enq 279 } 280 281 when (io.ptw.resp.fire()) { 282 v.zip(ptwResp_OldMatchVec).map{ case (vi, mi) => when (mi) { vi := false.B }} 283 } 284 285 counter := counter - do_deq + Mux(do_enq, enqNum, 0.U) 286 assert(counter <= Size.U, "counter should be no more than Size") 287 assert(inflight_counter <= Size.U, "inflight should be no more than Size") 288 when (counter === 0.U) { 289 assert(!io.ptw.req(0).fire(), "when counter is 0, should not req") 290 assert(isEmptyDeq && isEmptyIss, "when counter is 0, should be empty") 291 } 292 when (counter === Size.U) { 293 assert(mayFullDeq, "when counter is Size, should be full") 294 } 295 296 when (flush) { 297 v.map(_ := false.B) 298 deqPtr := 0.U 299 enqPtr := 0.U 300 issPtr := 0.U 301 ptwResp_valid := false.B 302 mayFullDeq := false.B 303 mayFullIss := false.B 304 counter := 0.U 305 inflight_counter := 0.U 306 } 307 308 // perf 309 XSPerfAccumulate("tlb_req_count", PopCount(Cat(io.tlb.req.map(_.valid)))) 310 XSPerfAccumulate("tlb_req_count_filtered", Mux(do_enq, accumEnqNum(Width - 1), 0.U)) 311 XSPerfAccumulate("ptw_req_count", io.ptw.req(0).fire()) 312 XSPerfAccumulate("ptw_req_cycle", inflight_counter) 313 XSPerfAccumulate("tlb_resp_count", io.tlb.resp.fire()) 314 XSPerfAccumulate("ptw_resp_count", io.ptw.resp.fire()) 315 XSPerfAccumulate("inflight_cycle", !isEmptyDeq) 316 for (i <- 0 until Size + 1) { 317 XSPerfAccumulate(s"counter${i}", counter === i.U) 318 } 319 320 for (i <- 0 until Size) { 321 TimeOutAssert(v(i), timeOutThreshold, s"Filter ${i} doesn't recv resp in time") 322 } 323} 324 325object PTWRepeater { 326 def apply(fenceDelay: Int, 327 tlb: TlbPtwIO, 328 sfence: SfenceBundle, 329 csr: TlbCsrBundle 330 )(implicit p: Parameters) = { 331 val width = tlb.req.size 332 val repeater = Module(new PTWRepeater(width, fenceDelay)) 333 repeater.io.apply(tlb, sfence, csr) 334 repeater 335 } 336 337 def apply(fenceDelay: Int, 338 tlb: TlbPtwIO, 339 ptw: TlbPtwIO, 340 sfence: SfenceBundle, 341 csr: TlbCsrBundle 342 )(implicit p: Parameters) = { 343 val width = tlb.req.size 344 val repeater = Module(new PTWRepeater(width, fenceDelay)) 345 repeater.io.apply(tlb, ptw, sfence, csr) 346 repeater 347 } 348} 349 350object PTWRepeaterNB { 351 def apply(passReady: Boolean, fenceDelay: Int, 352 tlb: TlbPtwIO, 353 sfence: SfenceBundle, 354 csr: TlbCsrBundle 355 )(implicit p: Parameters) = { 356 val width = tlb.req.size 357 val repeater = Module(new PTWRepeaterNB(width, passReady,fenceDelay)) 358 repeater.io.apply(tlb, sfence, csr) 359 repeater 360 } 361 362 def apply(passReady: Boolean, fenceDelay: Int, 363 tlb: TlbPtwIO, 364 ptw: TlbPtwIO, 365 sfence: SfenceBundle, 366 csr: TlbCsrBundle 367 )(implicit p: Parameters) = { 368 val width = tlb.req.size 369 val repeater = Module(new PTWRepeaterNB(width, passReady, fenceDelay)) 370 repeater.io.apply(tlb, ptw, sfence, csr) 371 repeater 372 } 373} 374 375object PTWFilter { 376 def apply(fenceDelay: Int, 377 tlb: VectorTlbPtwIO, 378 ptw: TlbPtwIO, 379 sfence: SfenceBundle, 380 csr: TlbCsrBundle, 381 size: Int 382 )(implicit p: Parameters) = { 383 val width = tlb.req.size 384 val filter = Module(new PTWFilter(width, size, fenceDelay)) 385 filter.io.apply(tlb, ptw, sfence, csr) 386 filter 387 } 388 389 def apply(fenceDelay: Int, 390 tlb: VectorTlbPtwIO, 391 sfence: SfenceBundle, 392 csr: TlbCsrBundle, 393 size: Int 394 )(implicit p: Parameters) = { 395 val width = tlb.req.size 396 val filter = Module(new PTWFilter(width, size, fenceDelay)) 397 filter.io.apply(tlb, sfence, csr) 398 filter 399 } 400 401} 402