1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.experimental.ExtModule 22import chisel3.util._ 23import chisel3.internal.naming.chiselName 24import xiangshan._ 25import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 26import utils._ 27import freechips.rocketchip.diplomacy.{IdRange, LazyModule, LazyModuleImp} 28import freechips.rocketchip.tilelink._ 29import xiangshan.backend.fu.{PMP, PMPChecker, PMPReqBundle, PMPRespBundle} 30import xiangshan.backend.fu.util.HasCSRConst 31 32class L2TLB()(implicit p: Parameters) extends LazyModule with HasPtwConst { 33 34 val node = TLClientNode(Seq(TLMasterPortParameters.v1( 35 clients = Seq(TLMasterParameters.v1( 36 "ptw", 37 sourceId = IdRange(0, MemReqWidth) 38 )) 39 ))) 40 41 lazy val module = new L2TLBImp(this) 42} 43 44@chiselName 45class L2TLBImp(outer: L2TLB)(implicit p: Parameters) extends PtwModule(outer) with HasCSRConst with HasPerfEvents { 46 47 val (mem, edge) = outer.node.out.head 48 49 val io = IO(new L2TLBIO) 50 val difftestIO = IO(new Bundle() { 51 val ptwResp = Output(Bool()) 52 val ptwAddr = Output(UInt(64.W)) 53 val ptwData = Output(Vec(4, UInt(64.W))) 54 }) 55 56 /* Ptw processes multiple requests 57 * Divide Ptw procedure into two stages: cache access ; mem access if cache miss 58 * miss queue itlb dtlb 59 * | | | 60 * ------arbiter------ 61 * | 62 * l1 - l2 - l3 - sp 63 * | 64 * ------------------------------------------- 65 * miss | queue | hit 66 * [][][][][][] | 67 * | | 68 * state machine accessing mem | 69 * | | 70 * ---------------arbiter--------------------- 71 * | | 72 * itlb dtlb 73 */ 74 75 difftestIO <> DontCare 76 77 val sfence = DelayN(io.sfence, 2) 78 val csr = DelayN(io.csr.tlb, 2) 79 val satp = csr.satp 80 val priv = csr.priv 81 val flush = sfence.valid || csr.satp.changed 82 83 val pmp = Module(new PMP()) 84 val pmp_check = VecInit(Seq.fill(2)(Module(new PMPChecker(lgMaxSize = 3, sameCycle = true)).io)) 85 pmp.io.distribute_csr := io.csr.distribute_csr 86 pmp_check.foreach(_.check_env.apply(ModeS, pmp.io.pmp, pmp.io.pma)) 87 88 val missQueue = Module(new L2TlbMissQueue) 89 val cache = Module(new PtwCache) 90 val ptw = Module(new PTW) 91 val llptw = Module(new LLPTW) 92 val arb1 = Module(new Arbiter(new PtwReq, PtwWidth)) 93 val arb2 = Module(new Arbiter(new Bundle { 94 val vpn = UInt(vpnLen.W) 95 val source = UInt(bSourceWidth.W) 96 }, if (l2tlbParams.enablePrefetch) 4 else 3)) 97 val outArb = (0 until PtwWidth).map(i => Module(new Arbiter(new PtwResp, 3)).io) 98 val outArbCachePort = 0 99 val outArbFsmPort = 1 100 val outArbMqPort = 2 101 102 // arb2 input port 103 val InArbPTWPort = 0 104 val InArbMissQueuePort = 1 105 val InArbTlbPort = 2 106 val InArbPrefetchPort = 3 107 // NOTE: when cache out but miss and ptw doesnt accept, 108 arb1.io.in <> VecInit(io.tlb.map(_.req(0))) 109 arb1.io.out.ready := arb2.io.in(InArbTlbPort).ready 110 111 arb2.io.in(InArbPTWPort).valid := ptw.io.llptw.valid 112 arb2.io.in(InArbPTWPort).bits.vpn := ptw.io.llptw.bits.req_info.vpn 113 arb2.io.in(InArbPTWPort).bits.source := ptw.io.llptw.bits.req_info.source 114 ptw.io.llptw.ready := arb2.io.in(InArbPTWPort).ready 115 block_decoupled(missQueue.io.out, arb2.io.in(InArbMissQueuePort), !ptw.io.req.ready) 116 arb2.io.in(InArbTlbPort).valid := arb1.io.out.valid 117 arb2.io.in(InArbTlbPort).bits.vpn := arb1.io.out.bits.vpn 118 arb2.io.in(InArbTlbPort).bits.source := arb1.io.chosen 119 if (l2tlbParams.enablePrefetch) { 120 val prefetch = Module(new L2TlbPrefetch()) 121 val recv = cache.io.resp 122 // NOTE: 1. prefetch doesn't gen prefetch 2. req from mq doesn't gen prefetch 123 // NOTE: 1. miss req gen prefetch 2. hit but prefetched gen prefetch 124 prefetch.io.in.valid := recv.fire() && !from_pre(recv.bits.req_info.source) && (!recv.bits.hit || 125 recv.bits.prefetch) && recv.bits.isFirst 126 prefetch.io.in.bits.vpn := recv.bits.req_info.vpn 127 prefetch.io.sfence := sfence 128 prefetch.io.csr := csr 129 arb2.io.in(InArbPrefetchPort) <> prefetch.io.out 130 } 131 arb2.io.out.ready := cache.io.req.ready 132 133 llptw.io.in.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && cache.io.resp.bits.toFsm.l2Hit && !cache.io.resp.bits.bypassed 134 llptw.io.in.bits.req_info := cache.io.resp.bits.req_info 135 llptw.io.in.bits.ppn := cache.io.resp.bits.toFsm.ppn 136 llptw.io.sfence := sfence 137 llptw.io.csr := csr 138 139 cache.io.req.valid := arb2.io.out.valid 140 cache.io.req.bits.req_info.vpn := arb2.io.out.bits.vpn 141 cache.io.req.bits.req_info.source := arb2.io.out.bits.source 142 cache.io.req.bits.isFirst := arb2.io.chosen =/= InArbMissQueuePort.U 143 cache.io.req.bits.bypassed.map(_ := false.B) 144 cache.io.sfence := sfence 145 cache.io.csr := csr 146 cache.io.resp.ready := Mux(cache.io.resp.bits.hit, 147 outReady(cache.io.resp.bits.req_info.source, outArbCachePort), 148 Mux(cache.io.resp.bits.toFsm.l2Hit && !cache.io.resp.bits.bypassed, llptw.io.in.ready, 149 Mux(cache.io.resp.bits.bypassed, missQueue.io.in.ready, missQueue.io.in.ready || ptw.io.req.ready))) 150 151 missQueue.io.in.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && 152 (!cache.io.resp.bits.toFsm.l2Hit || cache.io.resp.bits.bypassed) && 153 !from_pre(cache.io.resp.bits.req_info.source) && 154 (cache.io.resp.bits.bypassed || !ptw.io.req.ready) 155 missQueue.io.in.bits := cache.io.resp.bits.req_info 156 missQueue.io.sfence := sfence 157 missQueue.io.csr := csr 158 159 // NOTE: missQueue req has higher priority 160 ptw.io.req.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && !cache.io.resp.bits.toFsm.l2Hit && !cache.io.resp.bits.bypassed 161 ptw.io.req.bits.req_info := cache.io.resp.bits.req_info 162 ptw.io.req.bits.l1Hit := cache.io.resp.bits.toFsm.l1Hit 163 ptw.io.req.bits.ppn := cache.io.resp.bits.toFsm.ppn 164 ptw.io.csr := csr 165 ptw.io.sfence := sfence 166 ptw.io.resp.ready := outReady(ptw.io.resp.bits.source, outArbFsmPort) 167 168 // mem req 169 def blockBytes_align(addr: UInt) = { 170 Cat(addr(PAddrBits - 1, log2Up(l2tlbParams.blockBytes)), 0.U(log2Up(l2tlbParams.blockBytes).W)) 171 } 172 def addr_low_from_vpn(vpn: UInt) = { 173 vpn(log2Ceil(l2tlbParams.blockBytes)-log2Ceil(XLEN/8)-1, 0) 174 } 175 def addr_low_from_paddr(paddr: UInt) = { 176 paddr(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 177 } 178 def from_missqueue(id: UInt) = { 179 (id =/= l2tlbParams.llptwsize.U) 180 } 181 val waiting_resp = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B))) 182 val flush_latch = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B))) 183 for (i <- waiting_resp.indices) { 184 assert(!flush_latch(i) || waiting_resp(i)) // when sfence_latch wait for mem resp, waiting_resp should be true 185 } 186 187 val llptw_out = llptw.io.out 188 val llptw_mem = llptw.io.mem 189 llptw_mem.req_mask := waiting_resp.take(l2tlbParams.llptwsize) 190 ptw.io.mem.mask := waiting_resp.last 191 192 val mem_arb = Module(new Arbiter(new L2TlbMemReqBundle(), 2)) 193 mem_arb.io.in(0) <> ptw.io.mem.req 194 mem_arb.io.in(1) <> llptw_mem.req 195 mem_arb.io.out.ready := mem.a.ready && !flush 196 197 // assert, should not send mem access at same addr for twice. 198 val last_resp_vpn = RegEnable(cache.io.refill.bits.req_info.vpn, cache.io.refill.valid) 199 val last_resp_level = RegEnable(cache.io.refill.bits.level, cache.io.refill.valid) 200 val last_resp_v = RegInit(false.B) 201 val last_has_invalid = !Cat(cache.io.refill.bits.ptes.asTypeOf(Vec(blockBits/XLEN, UInt(XLEN.W))).map(a => a(0))).andR 202 when (cache.io.refill.valid) { last_resp_v := !last_has_invalid} 203 when (flush) { last_resp_v := false.B } 204 XSError(last_resp_v && cache.io.refill.valid && 205 (cache.io.refill.bits.req_info.vpn === last_resp_vpn) && 206 (cache.io.refill.bits.level === last_resp_level), 207 "l2tlb should not access mem at same addr for twice") 208 // ATTENTION: this may wronngly assert when: a ptes is l2, last part is valid, 209 // but the current part is invalid, so one more mem access happened 210 // If this happened, remove the assert. 211 212 val req_addr_low = Reg(Vec(MemReqWidth, UInt((log2Up(l2tlbParams.blockBytes)-log2Up(XLEN/8)).W))) 213 214 when (llptw.io.in.fire()) { 215 // when enq miss queue, set the req_addr_low to receive the mem resp data part 216 req_addr_low(llptw_mem.enq_ptr) := addr_low_from_vpn(llptw.io.in.bits.req_info.vpn) 217 } 218 when (mem_arb.io.out.fire()) { 219 req_addr_low(mem_arb.io.out.bits.id) := addr_low_from_paddr(mem_arb.io.out.bits.addr) 220 waiting_resp(mem_arb.io.out.bits.id) := true.B 221 } 222 // mem read 223 val memRead = edge.Get( 224 fromSource = mem_arb.io.out.bits.id, 225 // toAddress = memAddr(log2Up(CacheLineSize / 2 / 8) - 1, 0), 226 toAddress = blockBytes_align(mem_arb.io.out.bits.addr), 227 lgSize = log2Up(l2tlbParams.blockBytes).U 228 )._2 229 mem.a.bits := memRead 230 mem.a.valid := mem_arb.io.out.valid && !flush 231 mem.d.ready := true.B 232 // mem -> data buffer 233 val refill_data = Reg(Vec(blockBits / l1BusDataWidth, UInt(l1BusDataWidth.W))) 234 val refill_helper = edge.firstlastHelper(mem.d.bits, mem.d.fire()) 235 val mem_resp_done = refill_helper._3 236 val mem_resp_from_mq = from_missqueue(mem.d.bits.source) 237 when (mem.d.valid) { 238 assert(mem.d.bits.source <= l2tlbParams.llptwsize.U) 239 refill_data(refill_helper._4) := mem.d.bits.data 240 } 241 // save only one pte for each id 242 // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache) 243 val resp_pte = VecInit((0 until MemReqWidth).map(i => 244 if (i == l2tlbParams.llptwsize) {DataHoldBypass(get_part(refill_data, req_addr_low(i)), RegNext(mem_resp_done && !mem_resp_from_mq)) } 245 else { DataHoldBypass(get_part(refill_data, req_addr_low(i)), llptw_mem.buffer_it(i)) } 246 )) 247 248 // mem -> miss queue 249 llptw_mem.resp.valid := mem_resp_done && mem_resp_from_mq 250 llptw_mem.resp.bits.id := mem.d.bits.source 251 // mem -> ptw 252 ptw.io.mem.req.ready := mem.a.ready 253 ptw.io.mem.resp.valid := mem_resp_done && !mem_resp_from_mq 254 ptw.io.mem.resp.bits := resp_pte.last 255 // mem -> cache 256 val refill_from_mq = RegNext(mem_resp_from_mq) 257 cache.io.refill.valid := RegNext(mem_resp_done && !flush && !flush_latch(mem.d.bits.source)) 258 cache.io.refill.bits.ptes := refill_data.asUInt 259 cache.io.refill.bits.req_info := Mux(refill_from_mq, llptw_mem.refill, ptw.io.refill.req_info) 260 cache.io.refill.bits.level := Mux(refill_from_mq, 2.U, RegEnable(ptw.io.refill.level, 0.U, ptw.io.mem.req.fire())) 261 cache.io.refill.bits.addr_low := RegNext(req_addr_low(mem.d.bits.source)) 262 263 // pmp 264 pmp_check(0).req <> ptw.io.pmp.req 265 ptw.io.pmp.resp <> pmp_check(0).resp 266 pmp_check(1).req <> llptw.io.pmp.req 267 llptw.io.pmp.resp <> pmp_check(1).resp 268 269 llptw_out.ready := outReady(llptw_out.bits.req_info.source, outArbMqPort) 270 for (i <- 0 until PtwWidth) { 271 outArb(i).in(outArbCachePort).valid := cache.io.resp.valid && cache.io.resp.bits.hit && cache.io.resp.bits.req_info.source===i.U 272 outArb(i).in(outArbCachePort).bits.entry := cache.io.resp.bits.toTlb 273 outArb(i).in(outArbCachePort).bits.pf := !cache.io.resp.bits.toTlb.v 274 outArb(i).in(outArbCachePort).bits.af := false.B 275 outArb(i).in(outArbFsmPort).valid := ptw.io.resp.valid && ptw.io.resp.bits.source===i.U 276 outArb(i).in(outArbFsmPort).bits := ptw.io.resp.bits.resp 277 outArb(i).in(outArbMqPort).valid := llptw_out.valid && llptw_out.bits.req_info.source===i.U 278 outArb(i).in(outArbMqPort).bits := pte_to_ptwResp(resp_pte(llptw_out.bits.id), llptw_out.bits.req_info.vpn, llptw_out.bits.af, true) 279 } 280 281 // io.tlb.map(_.resp) <> outArb.map(_.out) 282 io.tlb.map(_.resp).zip(outArb.map(_.out)).map{ 283 case (resp, out) => resp <> out 284 } 285 286 // sfence 287 when (flush) { 288 for (i <- 0 until MemReqWidth) { 289 when (waiting_resp(i)) { 290 flush_latch(i) := true.B 291 } 292 } 293 } 294 // mem -> control signal 295 // waiting_resp and sfence_latch will be reset when mem_resp_done 296 when (mem_resp_done) { 297 waiting_resp(mem.d.bits.source) := false.B 298 flush_latch(mem.d.bits.source) := false.B 299 } 300 301 def block_decoupled[T <: Data](source: DecoupledIO[T], sink: DecoupledIO[T], block_signal: Bool) = { 302 sink.valid := source.valid && !block_signal 303 source.ready := sink.ready && !block_signal 304 sink.bits := source.bits 305 } 306 307 def get_part(data: Vec[UInt], index: UInt): UInt = { 308 val inner_data = data.asTypeOf(Vec(data.getWidth / XLEN, UInt(XLEN.W))) 309 inner_data(index) 310 } 311 312 def pte_to_ptwResp(pte: UInt, vpn: UInt, af: Bool, af_first: Boolean) : PtwResp = { 313 val pte_in = pte.asTypeOf(new PteBundle()) 314 val ptw_resp = Wire(new PtwResp()) 315 ptw_resp.entry.ppn := pte_in.ppn 316 ptw_resp.entry.level.map(_ := 2.U) 317 ptw_resp.entry.perm.map(_ := pte_in.getPerm()) 318 ptw_resp.entry.tag := vpn 319 ptw_resp.pf := (if (af_first) !af else true.B) && pte_in.isPf(2.U) 320 ptw_resp.af := (if (!af_first) pte_in.isPf(2.U) else true.B) && af 321 ptw_resp.entry.v := !ptw_resp.pf 322 ptw_resp.entry.prefetch := DontCare 323 ptw_resp.entry.asid := satp.asid 324 ptw_resp 325 } 326 327 def outReady(source: UInt, port: Int): Bool = { 328 MuxLookup(source, true.B, 329 (0 until PtwWidth).map(i => i.U -> outArb(i).in(port).ready)) 330 } 331 332 // debug info 333 for (i <- 0 until PtwWidth) { 334 XSDebug(p"[io.tlb(${i.U})] ${io.tlb(i)}\n") 335 } 336 XSDebug(p"[sfence] ${sfence}\n") 337 XSDebug(p"[io.csr.tlb] ${io.csr.tlb}\n") 338 339 for (i <- 0 until PtwWidth) { 340 XSPerfAccumulate(s"req_count${i}", io.tlb(i).req(0).fire()) 341 XSPerfAccumulate(s"req_blocked_count_${i}", io.tlb(i).req(0).valid && !io.tlb(i).req(0).ready) 342 } 343 XSPerfAccumulate(s"req_blocked_by_mq", arb1.io.out.valid && missQueue.io.out.valid) 344 for (i <- 0 until (MemReqWidth + 1)) { 345 XSPerfAccumulate(s"mem_req_util${i}", PopCount(waiting_resp) === i.U) 346 } 347 XSPerfAccumulate("mem_cycle", PopCount(waiting_resp) =/= 0.U) 348 XSPerfAccumulate("mem_count", mem.a.fire()) 349 350 // print configs 351 println(s"${l2tlbParams.name}: a ptw, a llptw with size ${l2tlbParams.llptwsize}, miss queue size ${MissQueueSize} l1:${l2tlbParams.l1Size} fa l2: nSets ${l2tlbParams.l2nSets} nWays ${l2tlbParams.l2nWays} l3: ${l2tlbParams.l3nSets} nWays ${l2tlbParams.l3nWays} blockBytes:${l2tlbParams.blockBytes}") 352 353 // time out assert 354 for (i <- 0 until MemReqWidth) { 355 TimeOutAssert(waiting_resp(i), timeOutThreshold, s"ptw mem resp time out wait_resp${i}") 356 TimeOutAssert(flush_latch(i), timeOutThreshold, s"ptw mem resp time out flush_latch${i}") 357 } 358 359 360 val perfEvents = Seq(llptw, cache, ptw).flatMap(_.getPerfEvents) 361 generatePerfEvent() 362} 363 364class PTEHelper() extends ExtModule { 365 val clock = IO(Input(Clock())) 366 val enable = IO(Input(Bool())) 367 val satp = IO(Input(UInt(64.W))) 368 val vpn = IO(Input(UInt(64.W))) 369 val pte = IO(Output(UInt(64.W))) 370 val level = IO(Output(UInt(8.W))) 371 val pf = IO(Output(UInt(8.W))) 372} 373 374class FakePTW()(implicit p: Parameters) extends XSModule with HasPtwConst { 375 val io = IO(new L2TLBIO) 376 377 for (i <- 0 until PtwWidth) { 378 io.tlb(i).req(0).ready := true.B 379 380 val helper = Module(new PTEHelper()) 381 helper.clock := clock 382 helper.enable := io.tlb(i).req(0).valid 383 helper.satp := io.csr.tlb.satp.ppn 384 helper.vpn := io.tlb(i).req(0).bits.vpn 385 val pte = helper.pte.asTypeOf(new PteBundle) 386 val level = helper.level 387 val pf = helper.pf 388 389 io.tlb(i).resp.valid := RegNext(io.tlb(i).req(0).valid) 390 assert(!io.tlb(i).resp.valid || io.tlb(i).resp.ready) 391 io.tlb(i).resp.bits.entry.tag := RegNext(io.tlb(i).req(0).bits.vpn) 392 io.tlb(i).resp.bits.entry.ppn := pte.ppn 393 io.tlb(i).resp.bits.entry.perm.map(_ := pte.getPerm()) 394 io.tlb(i).resp.bits.entry.level.map(_ := level) 395 io.tlb(i).resp.bits.pf := pf 396 io.tlb(i).resp.bits.af := DontCare // TODO: implement it 397 } 398} 399 400class L2TLBWrapper()(implicit p: Parameters) extends LazyModule with HasXSParameter { 401 val useSoftPTW = coreParams.softPTW 402 val node = if (!useSoftPTW) TLIdentityNode() else null 403 val ptw = if (!useSoftPTW) LazyModule(new L2TLB()) else null 404 if (!useSoftPTW) { 405 node := ptw.node 406 } 407 408 lazy val module = new LazyModuleImp(this) with HasPerfEvents { 409 val io = IO(new L2TLBIO) 410 val perfEvents = if (useSoftPTW) { 411 val fake_ptw = Module(new FakePTW()) 412 io <> fake_ptw.io 413 Seq() 414 } 415 else { 416 io <> ptw.module.io 417 ptw.module.getPerfEvents 418 } 419 generatePerfEvent() 420 } 421} 422