192e3bfefSLemover/*************************************************************************************** 292e3bfefSLemover* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 392e3bfefSLemover* Copyright (c) 2020-2021 Peng Cheng Laboratory 492e3bfefSLemover* 592e3bfefSLemover* XiangShan is licensed under Mulan PSL v2. 692e3bfefSLemover* You can use this software according to the terms and conditions of the Mulan PSL v2. 792e3bfefSLemover* You may obtain a copy of Mulan PSL v2 at: 892e3bfefSLemover* http://license.coscl.org.cn/MulanPSL2 992e3bfefSLemover* 1092e3bfefSLemover* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 1192e3bfefSLemover* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 1292e3bfefSLemover* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 1392e3bfefSLemover* 1492e3bfefSLemover* See the Mulan PSL v2 for more details. 1592e3bfefSLemover***************************************************************************************/ 1692e3bfefSLemover 1792e3bfefSLemoverpackage xiangshan.cache.mmu 1892e3bfefSLemover 198891a219SYinan Xuimport org.chipsalliance.cde.config.Parameters 2092e3bfefSLemoverimport chisel3._ 2192e3bfefSLemoverimport chisel3.experimental.ExtModule 2292e3bfefSLemoverimport chisel3.util._ 2392e3bfefSLemoverimport xiangshan._ 2492e3bfefSLemoverimport xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 2592e3bfefSLemoverimport utils._ 263c02ee8fSwakafaimport utility._ 2792e3bfefSLemoverimport freechips.rocketchip.diplomacy.{IdRange, LazyModule, LazyModuleImp} 2892e3bfefSLemoverimport freechips.rocketchip.tilelink._ 2992e3bfefSLemoverimport xiangshan.backend.fu.{PMP, PMPChecker, PMPReqBundle, PMPRespBundle} 3092e3bfefSLemoverimport xiangshan.backend.fu.util.HasCSRConst 319c26bab7SHaoyuan Fengimport difftest._ 3292e3bfefSLemover 3392e3bfefSLemoverclass L2TLB()(implicit p: Parameters) extends LazyModule with HasPtwConst { 3495e60e55STang Haojin override def shouldBeInlined: Boolean = false 3592e3bfefSLemover 3692e3bfefSLemover val node = TLClientNode(Seq(TLMasterPortParameters.v1( 3792e3bfefSLemover clients = Seq(TLMasterParameters.v1( 3892e3bfefSLemover "ptw", 3992e3bfefSLemover sourceId = IdRange(0, MemReqWidth) 40d2b20d1aSTang Haojin )), 41d2b20d1aSTang Haojin requestFields = Seq(ReqSourceField()) 4292e3bfefSLemover ))) 4392e3bfefSLemover 4492e3bfefSLemover lazy val module = new L2TLBImp(this) 4592e3bfefSLemover} 4692e3bfefSLemover 4792e3bfefSLemoverclass L2TLBImp(outer: L2TLB)(implicit p: Parameters) extends PtwModule(outer) with HasCSRConst with HasPerfEvents { 4892e3bfefSLemover 4992e3bfefSLemover val (mem, edge) = outer.node.out.head 5092e3bfefSLemover 5192e3bfefSLemover val io = IO(new L2TLBIO) 5292e3bfefSLemover val difftestIO = IO(new Bundle() { 5392e3bfefSLemover val ptwResp = Output(Bool()) 5492e3bfefSLemover val ptwAddr = Output(UInt(64.W)) 5592e3bfefSLemover val ptwData = Output(Vec(4, UInt(64.W))) 5692e3bfefSLemover }) 5792e3bfefSLemover 5892e3bfefSLemover /* Ptw processes multiple requests 5992e3bfefSLemover * Divide Ptw procedure into two stages: cache access ; mem access if cache miss 6092e3bfefSLemover * miss queue itlb dtlb 6192e3bfefSLemover * | | | 6292e3bfefSLemover * ------arbiter------ 6392e3bfefSLemover * | 6492e3bfefSLemover * l1 - l2 - l3 - sp 6592e3bfefSLemover * | 6692e3bfefSLemover * ------------------------------------------- 6792e3bfefSLemover * miss | queue | hit 6892e3bfefSLemover * [][][][][][] | 6992e3bfefSLemover * | | 7092e3bfefSLemover * state machine accessing mem | 7192e3bfefSLemover * | | 7292e3bfefSLemover * ---------------arbiter--------------------- 7392e3bfefSLemover * | | 7492e3bfefSLemover * itlb dtlb 7592e3bfefSLemover */ 7692e3bfefSLemover 7792e3bfefSLemover difftestIO <> DontCare 7892e3bfefSLemover 797797f035SbugGenerator val sfence_tmp = DelayN(io.sfence, 1) 807797f035SbugGenerator val csr_tmp = DelayN(io.csr.tlb, 1) 81d0de7e4aSpeixiaokun val sfence_dup = Seq.fill(9)(RegNext(sfence_tmp)) 82d0de7e4aSpeixiaokun val csr_dup = Seq.fill(8)(RegNext(csr_tmp)) 837797f035SbugGenerator val satp = csr_dup(0).satp 84d0de7e4aSpeixiaokun val vsatp = csr_dup(0).vsatp 85d0de7e4aSpeixiaokun val hgatp = csr_dup(0).hgatp 867797f035SbugGenerator val priv = csr_dup(0).priv 87d0de7e4aSpeixiaokun val flush = sfence_dup(0).valid || satp.changed || vsatp.changed || hgatp.changed 8892e3bfefSLemover 8992e3bfefSLemover val pmp = Module(new PMP()) 9092e3bfefSLemover val pmp_check = VecInit(Seq.fill(2)(Module(new PMPChecker(lgMaxSize = 3, sameCycle = true)).io)) 9192e3bfefSLemover pmp.io.distribute_csr := io.csr.distribute_csr 9292e3bfefSLemover pmp_check.foreach(_.check_env.apply(ModeS, pmp.io.pmp, pmp.io.pma)) 9392e3bfefSLemover 9492e3bfefSLemover val missQueue = Module(new L2TlbMissQueue) 9592e3bfefSLemover val cache = Module(new PtwCache) 9692e3bfefSLemover val ptw = Module(new PTW) 97d0de7e4aSpeixiaokun val hptw = Module(new HPTW) 9892e3bfefSLemover val llptw = Module(new LLPTW) 997797f035SbugGenerator val blockmq = Module(new BlockHelper(3)) 10092e3bfefSLemover val arb1 = Module(new Arbiter(new PtwReq, PtwWidth)) 10192e3bfefSLemover val arb2 = Module(new Arbiter(new Bundle { 10292e3bfefSLemover val vpn = UInt(vpnLen.W) 103d0de7e4aSpeixiaokun val s2xlate = UInt(2.W) 10492e3bfefSLemover val source = UInt(bSourceWidth.W) 105d0de7e4aSpeixiaokun }, if (l2tlbParams.enablePrefetch) 4 else 3 + if(l2tlbParams.HasHExtension) 1 else 0)) 106d0de7e4aSpeixiaokun val hptw_req_arb = Module(new Arbiter(new Bundle { 107d0de7e4aSpeixiaokun val id = UInt(log2Up(l2tlbParams.llptwsize).W) 108*82978df9Speixiaokun val gvpn = UInt(vpnLen.W) 109d0de7e4aSpeixiaokun }, 2)) 110d0de7e4aSpeixiaokun val hptw_resp_arb = Module(new Arbiter(new Bundle { 111d0de7e4aSpeixiaokun val resp = new HptwResp() 112d0de7e4aSpeixiaokun val id = UInt(log2Up(l2tlbParams.llptwsize).W) 113d0de7e4aSpeixiaokun }, 2)) 114d0de7e4aSpeixiaokun val outArb = (0 until PtwWidth).map(i => Module(new Arbiter(new Bundle { 115d0de7e4aSpeixiaokun val s2xlate = UInt(2.W) 116d0de7e4aSpeixiaokun val s1Resp = new PtwSectorResp () 117d0de7e4aSpeixiaokun val s2Resp = new HptwResp() 118d0de7e4aSpeixiaokun }, 1)).io) 119d0de7e4aSpeixiaokun val mergeArb = (0 until PtwWidth).map(i => Module(new Arbiter(new Bundle { 120d0de7e4aSpeixiaokun val s2xlate = UInt(2.W) 121d0de7e4aSpeixiaokun val s1Resp = new PtwMergeResp() 122d0de7e4aSpeixiaokun val s2Resp = new HptwResp() 123d0de7e4aSpeixiaokun }, 3)).io) 12492e3bfefSLemover val outArbCachePort = 0 12592e3bfefSLemover val outArbFsmPort = 1 12692e3bfefSLemover val outArbMqPort = 2 12792e3bfefSLemover 128d0de7e4aSpeixiaokun // hptw arb input port 129d0de7e4aSpeixiaokun val InHptwArbPTWPort = 0 130d0de7e4aSpeixiaokun val InHptwArbLLPTWPort = 1 131d0de7e4aSpeixiaokun hptw_req_arb.io.in(InHptwArbPTWPort).valid := ptw.io.hptw.req.valid 132d0de7e4aSpeixiaokun hptw_req_arb.io.in(InHptwArbPTWPort).bits.gvpn := ptw.io.hptw.req.bits.gvpn 133d0de7e4aSpeixiaokun hptw_req_arb.io.in(InHptwArbPTWPort).bits.id := ptw.io.hptw.req.bits.id 134d0de7e4aSpeixiaokun ptw.io.hptw.req.ready := hptw_req_arb.io.in(InHptwArbPTWPort).ready 135d0de7e4aSpeixiaokun 136d0de7e4aSpeixiaokun hptw_req_arb.io.in(InHptwArbLLPTWPort).valid := llptw.io.hptw.req.valid 137d0de7e4aSpeixiaokun hptw_req_arb.io.in(InHptwArbLLPTWPort).bits.gvpn := llptw.io.hptw.req.bits.gvpn 138d0de7e4aSpeixiaokun hptw_req_arb.io.in(InHptwArbLLPTWPort).bits.id := llptw.io.hptw.req.bits.id 139d0de7e4aSpeixiaokun llptw.io.hptw.req.ready := hptw_req_arb.io.in(InHptwArbLLPTWPort).ready 140d0de7e4aSpeixiaokun 141d0de7e4aSpeixiaokun val hptw_id = RegInit(0.U(log2Up(l2tlbParams.llptwsize).W)) 142d0de7e4aSpeixiaokun when(hptw_req_arb.io.out.valid) { 143d0de7e4aSpeixiaokun hptw_id := hptw_req_arb.io.out.bits.id 144d0de7e4aSpeixiaokun } 1459c503409SLemover // arb2 input port 1469c503409SLemover val InArbPTWPort = 0 1479c503409SLemover val InArbMissQueuePort = 1 1489c503409SLemover val InArbTlbPort = 2 1499c503409SLemover val InArbPrefetchPort = 3 150d0de7e4aSpeixiaokun val InArbHPTWPort = 4 15192e3bfefSLemover // NOTE: when cache out but miss and ptw doesnt accept, 15292e3bfefSLemover arb1.io.in <> VecInit(io.tlb.map(_.req(0))) 1539c503409SLemover arb1.io.out.ready := arb2.io.in(InArbTlbPort).ready 15492e3bfefSLemover 1559c503409SLemover arb2.io.in(InArbPTWPort).valid := ptw.io.llptw.valid 1569c503409SLemover arb2.io.in(InArbPTWPort).bits.vpn := ptw.io.llptw.bits.req_info.vpn 157d0de7e4aSpeixiaokun arb2.io.in(InArbPTWPort).bits.s2xlate := ptw.io.llptw.bits.req_info.s2xlate 1589c503409SLemover arb2.io.in(InArbPTWPort).bits.source := ptw.io.llptw.bits.req_info.source 1599c503409SLemover ptw.io.llptw.ready := arb2.io.in(InArbPTWPort).ready 16092e3bfefSLemover block_decoupled(missQueue.io.out, arb2.io.in(InArbMissQueuePort), !ptw.io.req.ready) 1617797f035SbugGenerator 16292e3bfefSLemover arb2.io.in(InArbTlbPort).valid := arb1.io.out.valid 16392e3bfefSLemover arb2.io.in(InArbTlbPort).bits.vpn := arb1.io.out.bits.vpn 164d0de7e4aSpeixiaokun arb2.io.in(InArbTlbPort).bits.s2xlate := arb1.io.out.bits.s2xlate 16592e3bfefSLemover arb2.io.in(InArbTlbPort).bits.source := arb1.io.chosen 166d0de7e4aSpeixiaokun 167d0de7e4aSpeixiaokun arb2.io.in(InArbHPTWPort).valid := hptw_req_arb.io.out.valid 168*82978df9Speixiaokun arb2.io.in(InArbHPTWPort).bits.vpn := hptw_req_arb.io.out.bits.gvpn 169*82978df9Speixiaokun arb2.io.in(InArbHPTWPort).bits.s2xlate := onlyStage2 170d0de7e4aSpeixiaokun arb2.io.in(InArbHPTWPort).bits.source := DontCare 17192e3bfefSLemover if (l2tlbParams.enablePrefetch) { 17292e3bfefSLemover val prefetch = Module(new L2TlbPrefetch()) 17392e3bfefSLemover val recv = cache.io.resp 17492e3bfefSLemover // NOTE: 1. prefetch doesn't gen prefetch 2. req from mq doesn't gen prefetch 17592e3bfefSLemover // NOTE: 1. miss req gen prefetch 2. hit but prefetched gen prefetch 176935edac4STang Haojin prefetch.io.in.valid := recv.fire && !from_pre(recv.bits.req_info.source) && (!recv.bits.hit || 17792e3bfefSLemover recv.bits.prefetch) && recv.bits.isFirst 17892e3bfefSLemover prefetch.io.in.bits.vpn := recv.bits.req_info.vpn 1797797f035SbugGenerator prefetch.io.sfence := sfence_dup(0) 1807797f035SbugGenerator prefetch.io.csr := csr_dup(0) 18192e3bfefSLemover arb2.io.in(InArbPrefetchPort) <> prefetch.io.out 1825afdf73cSHaoyuan Feng 183da3bf434SMaxpicca-Li val isWriteL2TlbPrefetchTable = WireInit(Constantin.createRecord("isWriteL2TlbPrefetchTable" + p(XSCoreParamsKey).HartId.toString)) 1845afdf73cSHaoyuan Feng val L2TlbPrefetchTable = ChiselDB.createTable("L2TlbPrefetch_hart" + p(XSCoreParamsKey).HartId.toString, new L2TlbPrefetchDB) 1855afdf73cSHaoyuan Feng val L2TlbPrefetchDB = Wire(new L2TlbPrefetchDB) 1865afdf73cSHaoyuan Feng L2TlbPrefetchDB.vpn := prefetch.io.out.bits.vpn 187da3bf434SMaxpicca-Li L2TlbPrefetchTable.log(L2TlbPrefetchDB, isWriteL2TlbPrefetchTable.orR && prefetch.io.out.fire, "L2TlbPrefetch", clock, reset) 18892e3bfefSLemover } 18992e3bfefSLemover arb2.io.out.ready := cache.io.req.ready 19092e3bfefSLemover 1917797f035SbugGenerator 1927797f035SbugGenerator val mq_arb = Module(new Arbiter(new L2TlbInnerBundle, 2)) 1937797f035SbugGenerator mq_arb.io.in(0).valid := cache.io.resp.valid && !cache.io.resp.bits.hit && 1947797f035SbugGenerator (!cache.io.resp.bits.toFsm.l2Hit || cache.io.resp.bits.bypassed) && 1957797f035SbugGenerator !from_pre(cache.io.resp.bits.req_info.source) && 1967797f035SbugGenerator (cache.io.resp.bits.bypassed || cache.io.resp.bits.isFirst || !ptw.io.req.ready) 1977797f035SbugGenerator mq_arb.io.in(0).bits := cache.io.resp.bits.req_info 1987797f035SbugGenerator mq_arb.io.in(1) <> llptw.io.cache 1997797f035SbugGenerator missQueue.io.in <> mq_arb.io.out 2007797f035SbugGenerator missQueue.io.sfence := sfence_dup(6) 2017797f035SbugGenerator missQueue.io.csr := csr_dup(5) 2027797f035SbugGenerator 2037797f035SbugGenerator blockmq.io.start := missQueue.io.out.fire 204935edac4STang Haojin blockmq.io.enable := ptw.io.req.fire 2057797f035SbugGenerator 206d0de7e4aSpeixiaokun llptw.io.in.valid := cache.io.resp.valid && 207d0de7e4aSpeixiaokun !cache.io.resp.bits.hit && 208d0de7e4aSpeixiaokun cache.io.resp.bits.toFsm.l2Hit && 209d0de7e4aSpeixiaokun !cache.io.resp.bits.bypassed && 210d0de7e4aSpeixiaokun !cache.io.resp.bits.isHptw 2119c503409SLemover llptw.io.in.bits.req_info := cache.io.resp.bits.req_info 2129c503409SLemover llptw.io.in.bits.ppn := cache.io.resp.bits.toFsm.ppn 2137797f035SbugGenerator llptw.io.sfence := sfence_dup(1) 2147797f035SbugGenerator llptw.io.csr := csr_dup(1) 21592e3bfefSLemover 21692e3bfefSLemover cache.io.req.valid := arb2.io.out.valid 21792e3bfefSLemover cache.io.req.bits.req_info.vpn := arb2.io.out.bits.vpn 218d0de7e4aSpeixiaokun cache.io.req.bits.req_info.s2xlate := arb2.io.out.bits.s2xlate 21992e3bfefSLemover cache.io.req.bits.req_info.source := arb2.io.out.bits.source 22092e3bfefSLemover cache.io.req.bits.isFirst := arb2.io.chosen =/= InArbMissQueuePort.U 221d0de7e4aSpeixiaokun cache.io.req.bits.isHptw := arb2.io.chosen === InArbHPTWPort.U 222d0de7e4aSpeixiaokun cache.io.req.bits.hptwId := hptw_id 2231f4a7c0cSLemover cache.io.req.bits.bypassed.map(_ := false.B) 2247797f035SbugGenerator cache.io.sfence := sfence_dup(2) 2257797f035SbugGenerator cache.io.csr := csr_dup(2) 2267797f035SbugGenerator cache.io.sfence_dup.zip(sfence_dup.drop(2).take(4)).map(s => s._1 := s._2) 2277797f035SbugGenerator cache.io.csr_dup.zip(csr_dup.drop(2).take(3)).map(c => c._1 := c._2) 22892e3bfefSLemover cache.io.resp.ready := Mux(cache.io.resp.bits.hit, 22992e3bfefSLemover outReady(cache.io.resp.bits.req_info.source, outArbCachePort), 2309c503409SLemover Mux(cache.io.resp.bits.toFsm.l2Hit && !cache.io.resp.bits.bypassed, llptw.io.in.ready, 2317797f035SbugGenerator Mux(cache.io.resp.bits.bypassed || cache.io.resp.bits.isFirst, mq_arb.io.in(0).ready, mq_arb.io.in(0).ready || ptw.io.req.ready))) 23292e3bfefSLemover 23392e3bfefSLemover // NOTE: missQueue req has higher priority 2347797f035SbugGenerator ptw.io.req.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && !cache.io.resp.bits.toFsm.l2Hit && 2357797f035SbugGenerator !cache.io.resp.bits.bypassed && 236d0de7e4aSpeixiaokun !cache.io.resp.bits.isFirst && 237d0de7e4aSpeixiaokun !cache.io.resp.bits.isHptw 23892e3bfefSLemover ptw.io.req.bits.req_info := cache.io.resp.bits.req_info 23992e3bfefSLemover ptw.io.req.bits.l1Hit := cache.io.resp.bits.toFsm.l1Hit 24092e3bfefSLemover ptw.io.req.bits.ppn := cache.io.resp.bits.toFsm.ppn 2417797f035SbugGenerator ptw.io.sfence := sfence_dup(7) 2427797f035SbugGenerator ptw.io.csr := csr_dup(6) 24392e3bfefSLemover ptw.io.resp.ready := outReady(ptw.io.resp.bits.source, outArbFsmPort) 24492e3bfefSLemover 245d0de7e4aSpeixiaokun hptw.io.req.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && !cache.io.resp.bits.bypassed & cache.io.resp.bits.isHptw 246*82978df9Speixiaokun hptw.io.req.bits.gvpn := cache.io.resp.bits.req_info.vpn 247d0de7e4aSpeixiaokun hptw.io.req.bits.id := cache.io.resp.bits.toHptw.id 248d0de7e4aSpeixiaokun hptw.io.req.bits.l1Hit := cache.io.resp.bits.toHptw.l1Hit 249d0de7e4aSpeixiaokun hptw.io.req.bits.l2Hit := cache.io.resp.bits.toHptw.l2Hit 250d0de7e4aSpeixiaokun hptw.io.sfence := sfence_dup(8) 251d0de7e4aSpeixiaokun hptw.io.csr := csr_dup(7) 25292e3bfefSLemover // mem req 25392e3bfefSLemover def blockBytes_align(addr: UInt) = { 25492e3bfefSLemover Cat(addr(PAddrBits - 1, log2Up(l2tlbParams.blockBytes)), 0.U(log2Up(l2tlbParams.blockBytes).W)) 25592e3bfefSLemover } 25692e3bfefSLemover def addr_low_from_vpn(vpn: UInt) = { 25792e3bfefSLemover vpn(log2Ceil(l2tlbParams.blockBytes)-log2Ceil(XLEN/8)-1, 0) 25892e3bfefSLemover } 25992e3bfefSLemover def addr_low_from_paddr(paddr: UInt) = { 26092e3bfefSLemover paddr(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 26192e3bfefSLemover } 262d0de7e4aSpeixiaokun def from_llptw(id: UInt) = { 263d0de7e4aSpeixiaokun id < l2tlbParams.llptwsize.U 264d0de7e4aSpeixiaokun } 265d0de7e4aSpeixiaokun def from_ptw(id: UInt) = { 266d0de7e4aSpeixiaokun id === l2tlbParams.llptwsize.U 267d0de7e4aSpeixiaokun } 268d0de7e4aSpeixiaokun def from_hptw(id: UInt) = { 269d0de7e4aSpeixiaokun id === l2tlbParams.llptwsize.U + 1.U 27092e3bfefSLemover } 27192e3bfefSLemover val waiting_resp = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B))) 27292e3bfefSLemover val flush_latch = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B))) 27392e3bfefSLemover for (i <- waiting_resp.indices) { 27492e3bfefSLemover assert(!flush_latch(i) || waiting_resp(i)) // when sfence_latch wait for mem resp, waiting_resp should be true 27592e3bfefSLemover } 27692e3bfefSLemover 27792e3bfefSLemover val llptw_out = llptw.io.out 27892e3bfefSLemover val llptw_mem = llptw.io.mem 27992e3bfefSLemover llptw_mem.req_mask := waiting_resp.take(l2tlbParams.llptwsize) 280d0de7e4aSpeixiaokun ptw.io.mem.mask := waiting_resp.slice(l2tlbParams.llptwsize, l2tlbParams.llptwsize+1) 281d0de7e4aSpeixiaokun hptw.io.mem.mask := waiting_resp.slice(l2tlbParams.llptwsize+1, l2tlbParams.llptwsize+2) 28292e3bfefSLemover 283d0de7e4aSpeixiaokun val mem_arb = Module(new Arbiter(new L2TlbMemReqBundle(), 3)) 28492e3bfefSLemover mem_arb.io.in(0) <> ptw.io.mem.req 28592e3bfefSLemover mem_arb.io.in(1) <> llptw_mem.req 286d0de7e4aSpeixiaokun mem_arb.io.in(2) <> hptw.io.mem.req 28792e3bfefSLemover mem_arb.io.out.ready := mem.a.ready && !flush 28892e3bfefSLemover 2891f4a7c0cSLemover // assert, should not send mem access at same addr for twice. 2907797f035SbugGenerator val last_resp_vpn = RegEnable(cache.io.refill.bits.req_info_dup(0).vpn, cache.io.refill.valid) 2917797f035SbugGenerator val last_resp_level = RegEnable(cache.io.refill.bits.level_dup(0), cache.io.refill.valid) 2921f4a7c0cSLemover val last_resp_v = RegInit(false.B) 293dd7fe201SHaoyuan Feng val last_has_invalid = !Cat(cache.io.refill.bits.ptes.asTypeOf(Vec(blockBits/XLEN, UInt(XLEN.W))).map(a => a(0))).andR || cache.io.refill.bits.sel_pte_dup(0).asTypeOf(new PteBundle).isAf() 2941f4a7c0cSLemover when (cache.io.refill.valid) { last_resp_v := !last_has_invalid} 2951f4a7c0cSLemover when (flush) { last_resp_v := false.B } 2961f4a7c0cSLemover XSError(last_resp_v && cache.io.refill.valid && 2977797f035SbugGenerator (cache.io.refill.bits.req_info_dup(0).vpn === last_resp_vpn) && 2987797f035SbugGenerator (cache.io.refill.bits.level_dup(0) === last_resp_level), 2991f4a7c0cSLemover "l2tlb should not access mem at same addr for twice") 300d0de7e4aSpeixiaokun // ATTENTION: this may wrongly assert when: a ptes is l2, last part is valid, 3011f4a7c0cSLemover // but the current part is invalid, so one more mem access happened 3021f4a7c0cSLemover // If this happened, remove the assert. 3031f4a7c0cSLemover 30492e3bfefSLemover val req_addr_low = Reg(Vec(MemReqWidth, UInt((log2Up(l2tlbParams.blockBytes)-log2Up(XLEN/8)).W))) 30592e3bfefSLemover 306935edac4STang Haojin when (llptw.io.in.fire) { 30792e3bfefSLemover // when enq miss queue, set the req_addr_low to receive the mem resp data part 30892e3bfefSLemover req_addr_low(llptw_mem.enq_ptr) := addr_low_from_vpn(llptw.io.in.bits.req_info.vpn) 30992e3bfefSLemover } 310935edac4STang Haojin when (mem_arb.io.out.fire) { 31192e3bfefSLemover req_addr_low(mem_arb.io.out.bits.id) := addr_low_from_paddr(mem_arb.io.out.bits.addr) 31292e3bfefSLemover waiting_resp(mem_arb.io.out.bits.id) := true.B 31392e3bfefSLemover } 31492e3bfefSLemover // mem read 31592e3bfefSLemover val memRead = edge.Get( 31692e3bfefSLemover fromSource = mem_arb.io.out.bits.id, 31792e3bfefSLemover // toAddress = memAddr(log2Up(CacheLineSize / 2 / 8) - 1, 0), 31892e3bfefSLemover toAddress = blockBytes_align(mem_arb.io.out.bits.addr), 31992e3bfefSLemover lgSize = log2Up(l2tlbParams.blockBytes).U 32092e3bfefSLemover )._2 32192e3bfefSLemover mem.a.bits := memRead 32292e3bfefSLemover mem.a.valid := mem_arb.io.out.valid && !flush 323d2b20d1aSTang Haojin mem.a.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.PTW.id.U) 32492e3bfefSLemover mem.d.ready := true.B 32592e3bfefSLemover // mem -> data buffer 32692e3bfefSLemover val refill_data = Reg(Vec(blockBits / l1BusDataWidth, UInt(l1BusDataWidth.W))) 327935edac4STang Haojin val refill_helper = edge.firstlastHelper(mem.d.bits, mem.d.fire) 32892e3bfefSLemover val mem_resp_done = refill_helper._3 329d0de7e4aSpeixiaokun val mem_resp_from_llptw = from_llptw(mem.d.bits.source) 330d0de7e4aSpeixiaokun val mem_resp_from_ptw = from_ptw(mem.d.bits.source) 331d0de7e4aSpeixiaokun val mem_resp_from_hptw = from_hptw(mem.d.bits.source) 33292e3bfefSLemover when (mem.d.valid) { 333d0de7e4aSpeixiaokun assert(mem.d.bits.source < MemReqWidth.U) 33492e3bfefSLemover refill_data(refill_helper._4) := mem.d.bits.data 33592e3bfefSLemover } 3367797f035SbugGenerator // refill_data_tmp is the wire fork of refill_data, but one cycle earlier 3377797f035SbugGenerator val refill_data_tmp = WireInit(refill_data) 3387797f035SbugGenerator refill_data_tmp(refill_helper._4) := mem.d.bits.data 3397797f035SbugGenerator 34092e3bfefSLemover // save only one pte for each id 34192e3bfefSLemover // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache) 34292e3bfefSLemover val resp_pte = VecInit((0 until MemReqWidth).map(i => 343d0de7e4aSpeixiaokun if (i == l2tlbParams.llptwsize + 1) {RegEnable(get_part(refill_data_tmp, req_addr_low(i)), mem_resp_done && mem_resp_from_hptw) } 344d0de7e4aSpeixiaokun else if (i == l2tlbParams.llptwsize) {RegEnable(get_part(refill_data_tmp, req_addr_low(i)), mem_resp_done && mem_resp_from_ptw) } 34592e3bfefSLemover else { DataHoldBypass(get_part(refill_data, req_addr_low(i)), llptw_mem.buffer_it(i)) } 3467797f035SbugGenerator // llptw could not use refill_data_tmp, because enq bypass's result works at next cycle 34792e3bfefSLemover )) 34892e3bfefSLemover 34963632028SHaoyuan Feng // save eight ptes for each id when sector tlb 35063632028SHaoyuan Feng // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache) 35163632028SHaoyuan Feng val resp_pte_sector = VecInit((0 until MemReqWidth).map(i => 352d0de7e4aSpeixiaokun if (i == l2tlbParams.llptwsize + 1) {RegEnable(refill_data_tmp, mem_resp_done && mem_resp_from_hptw) } 353d0de7e4aSpeixiaokun else if (i == l2tlbParams.llptwsize) {RegEnable(refill_data_tmp, mem_resp_done && mem_resp_from_ptw) } 35463632028SHaoyuan Feng else { DataHoldBypass(refill_data, llptw_mem.buffer_it(i)) } 35563632028SHaoyuan Feng // llptw could not use refill_data_tmp, because enq bypass's result works at next cycle 35663632028SHaoyuan Feng )) 35763632028SHaoyuan Feng 358d0de7e4aSpeixiaokun // mem -> llptw 359d0de7e4aSpeixiaokun llptw_mem.resp.valid := mem_resp_done && mem_resp_from_llptw 3607797f035SbugGenerator llptw_mem.resp.bits.id := DataHoldBypass(mem.d.bits.source, mem.d.valid) 36192e3bfefSLemover // mem -> ptw 36292e3bfefSLemover ptw.io.mem.req.ready := mem.a.ready 363d0de7e4aSpeixiaokun ptw.io.mem.resp.valid := mem_resp_done && mem_resp_from_ptw 364d0de7e4aSpeixiaokun ptw.io.mem.resp.bits := resp_pte.slice(l2tlbParams.llptwsize, l2tlbParams.llptwsize + 1) 365d0de7e4aSpeixiaokun // mem -> hptw 366d0de7e4aSpeixiaokun hptw.io.mem.req.ready := mem.a.ready 367d0de7e4aSpeixiaokun hptw.io.mem.resp.valid := mem_resp_done && mem_resp_from_hptw 368d0de7e4aSpeixiaokun hptw.io.mem.resp.bits := resp_pte.slice(l2tlbParams.llptwsize + 1, l2tlbParams.llptwsize + 2) 36992e3bfefSLemover // mem -> cache 370d0de7e4aSpeixiaokun val refill_from_llptw = mem_resp_from_llptw 371d0de7e4aSpeixiaokun val refill_from_ptw = mem_resp_from_ptw 372d0de7e4aSpeixiaokun val refill_from_hptw = mem_resp_from_hptw 373d0de7e4aSpeixiaokun val refill_level = Mux(refill_from_llptw, 2.U, Mux(refill_from_ptw, RegEnable(ptw.io.refill.level, init = 0.U, ptw.io.mem.req.fire()), RegEnable(hptw.io.refill.level, init = 0.U, hptw.io.mem.req.fire()))) 3747797f035SbugGenerator val refill_valid = mem_resp_done && !flush && !flush_latch(mem.d.bits.source) 3757797f035SbugGenerator 3767797f035SbugGenerator cache.io.refill.valid := RegNext(refill_valid, false.B) 37792e3bfefSLemover cache.io.refill.bits.ptes := refill_data.asUInt 378d0de7e4aSpeixiaokun cache.io.refill.bits.req_info_dup.map(_ := RegEnable(Mux(refill_from_llptw, llptw_mem.refill, Mux(refill_from_ptw, ptw.io.refill.req_info, hptw.io.refill.req_info)), refill_valid)) 3797797f035SbugGenerator cache.io.refill.bits.level_dup.map(_ := RegEnable(refill_level, refill_valid)) 3807797f035SbugGenerator cache.io.refill.bits.levelOH(refill_level, refill_valid) 3817797f035SbugGenerator cache.io.refill.bits.sel_pte_dup.map(_ := RegNext(sel_data(refill_data_tmp.asUInt, req_addr_low(mem.d.bits.source)))) 38292e3bfefSLemover 3839c26bab7SHaoyuan Feng if (env.EnableDifftest) { 3849c26bab7SHaoyuan Feng val difftest_ptw_addr = RegInit(VecInit(Seq.fill(MemReqWidth)(0.U(PAddrBits.W)))) 3859c26bab7SHaoyuan Feng when (mem.a.valid) { 3869c26bab7SHaoyuan Feng difftest_ptw_addr(mem.a.bits.source) := mem.a.bits.address 3879c26bab7SHaoyuan Feng } 3889c26bab7SHaoyuan Feng 389a0c65233SYinan Xu val difftest = DifftestModule(new DiffRefillEvent, dontCare = true) 390254e4960SHaoyuan Feng difftest.coreid := io.hartId 3917d45a146SYinan Xu difftest.index := 2.U 3927d45a146SYinan Xu difftest.valid := cache.io.refill.valid 3937d45a146SYinan Xu difftest.addr := difftest_ptw_addr(RegNext(mem.d.bits.source)) 3947d45a146SYinan Xu difftest.data := refill_data.asTypeOf(difftest.data) 395935edac4STang Haojin difftest.idtfr := DontCare 3969c26bab7SHaoyuan Feng } 3979c26bab7SHaoyuan Feng 3985ab1b84dSHaoyuan Feng if (env.EnableDifftest) { 3995ab1b84dSHaoyuan Feng for (i <- 0 until PtwWidth) { 4007d45a146SYinan Xu val difftest = DifftestModule(new DiffL2TLBEvent) 401254e4960SHaoyuan Feng difftest.coreid := io.hartId 4027d45a146SYinan Xu difftest.valid := io.tlb(i).resp.fire && !io.tlb(i).resp.bits.af 4037d45a146SYinan Xu difftest.index := i.U 4047d45a146SYinan Xu difftest.satp := io.csr.tlb.satp.ppn 4057d45a146SYinan Xu difftest.vpn := Cat(io.tlb(i).resp.bits.entry.tag, 0.U(sectortlbwidth.W)) 40663632028SHaoyuan Feng for (j <- 0 until tlbcontiguous) { 4077d45a146SYinan Xu difftest.ppn(j) := Cat(io.tlb(i).resp.bits.entry.ppn, io.tlb(i).resp.bits.ppn_low(j)) 4087d45a146SYinan Xu difftest.valididx(j) := io.tlb(i).resp.bits.valididx(j) 409*82978df9Speixiaokun difftest.io.pteidx(j) := io.tlb(i).resp.bits.s1.pteidx(j) 41063632028SHaoyuan Feng } 4117d45a146SYinan Xu difftest.perm := io.tlb(i).resp.bits.entry.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt 4127d45a146SYinan Xu difftest.level := io.tlb(i).resp.bits.entry.level.getOrElse(0.U.asUInt) 4137d45a146SYinan Xu difftest.pf := io.tlb(i).resp.bits.pf 4145ab1b84dSHaoyuan Feng } 4155ab1b84dSHaoyuan Feng } 4165ab1b84dSHaoyuan Feng 41792e3bfefSLemover // pmp 41892e3bfefSLemover pmp_check(0).req <> ptw.io.pmp.req 41992e3bfefSLemover ptw.io.pmp.resp <> pmp_check(0).resp 42092e3bfefSLemover pmp_check(1).req <> llptw.io.pmp.req 42192e3bfefSLemover llptw.io.pmp.resp <> pmp_check(1).resp 42292e3bfefSLemover 42392e3bfefSLemover llptw_out.ready := outReady(llptw_out.bits.req_info.source, outArbMqPort) 42463632028SHaoyuan Feng 425d0de7e4aSpeixiaokun // hptw and page cache -> ptw and llptw 426d0de7e4aSpeixiaokun val HptwRespArbCachePort = 0 427d0de7e4aSpeixiaokun val HptwRespArbHptw = 0 428d0de7e4aSpeixiaokun hptw_resp_arb.io.in(HptwRespArbCachePort).valid := cache.io.resp.valid && cache.io.resp.bits.hit && cache.io.resp.bits.isHptw 429d0de7e4aSpeixiaokun hptw_resp_arb.io.in(HptwRespArbCachePort).bits.id := cache.io.resp.bits.toHptw.id 430d0de7e4aSpeixiaokun hptw_resp_arb.io.in(HptwRespArbCachePort).bits.resp := cache.io.resp.bits.toHptw.resp 431d0de7e4aSpeixiaokun hptw_resp_arb.io.in(HptwRespArbHptw).valid := hptw.io.resp.valid 432d0de7e4aSpeixiaokun hptw_resp_arb.io.in(HptwRespArbHptw).bits.id := hptw.io.resp.bits.id 433d0de7e4aSpeixiaokun hptw_resp_arb.io.in(HptwRespArbHptw).bits.resp := hptw.io.resp.bits.resp 434d0de7e4aSpeixiaokun 435d0de7e4aSpeixiaokun ptw.io.hptw.resp.valid := hptw_resp_arb.io.out.valid && hptw_resp_arb.io.out.bits.id === FsmReqID.U 436d0de7e4aSpeixiaokun ptw.io.hptw.resp.bits.h_resp := hptw_resp_arb.io.out.bits.resp 437d0de7e4aSpeixiaokun llptw.io.hptw.resp.valid := hptw_resp_arb.io.out.valid && hptw_resp_arb.io.out.bits.id =/= FsmReqID.U 438d0de7e4aSpeixiaokun llptw.io.hptw.resp.bits.h_resp := hptw_resp_arb.io.out.bits.resp 439d0de7e4aSpeixiaokun 44063632028SHaoyuan Feng // Timing: Maybe need to do some optimization or even add one more cycle 44192e3bfefSLemover for (i <- 0 until PtwWidth) { 44263632028SHaoyuan Feng mergeArb(i).in(outArbCachePort).valid := cache.io.resp.valid && cache.io.resp.bits.hit && cache.io.resp.bits.req_info.source===i.U 443d0de7e4aSpeixiaokun mergeArb(i).in(outArbCachePort).bits.s2xlate := cache.io.resp.bits.req_info.s2xlate 444d0de7e4aSpeixiaokun mergeArb(i).in(outArbCachePort).bits.s1Resp := cache.io.resp.bits.toTlb 445d0de7e4aSpeixiaokun mergeArb(i).in(outArbCachePort).bits.s2Resp := cache.io.resp.bits.toHptw.resp 44663632028SHaoyuan Feng mergeArb(i).in(outArbFsmPort).valid := ptw.io.resp.valid && ptw.io.resp.bits.source===i.U 447d0de7e4aSpeixiaokun mergeArb(i).in(outArbFsmPort).bits.s2xlate := ptw.io.resp.bits.s2xlate 448d0de7e4aSpeixiaokun mergeArb(i).in(outArbFsmPort).bits.s1Resp := ptw.io.resp.bits.resp 449d0de7e4aSpeixiaokun mergeArb(i).in(outArbFsmPort).bits.s2Resp := ptw.io.resp.bits.h_resp 45063632028SHaoyuan Feng mergeArb(i).in(outArbMqPort).valid := llptw_out.valid && llptw_out.bits.req_info.source===i.U 451d0de7e4aSpeixiaokun mergeArb(i).in(outArbMqPort).bits.s2xlate := llptw_out.bits.req_info.s2xlate 452d0de7e4aSpeixiaokun mergeArb(i).in(outArbMqPort).bits.s1Resp := contiguous_pte_to_merge_ptwResp(resp_pte_sector(llptw_out.bits.id).asUInt, llptw_out.bits.req_info.vpn, llptw_out.bits.af, true) 453d0de7e4aSpeixiaokun mergeArb(i).in(outArbMqPort).bits.s2Resp := llptw_out.bits.h_resp 45463632028SHaoyuan Feng mergeArb(i).out.ready := outArb(i).in(0).ready 45563632028SHaoyuan Feng } 45663632028SHaoyuan Feng 45763632028SHaoyuan Feng for (i <- 0 until PtwWidth) { 45863632028SHaoyuan Feng outArb(i).in(0).valid := mergeArb(i).out.valid 459d0de7e4aSpeixiaokun outArb(i).in(0).bits.s1Resp := merge_ptwResp_to_sector_ptwResp(mergeArb(i).out.bits.s1Resp) 460d0de7e4aSpeixiaokun outArb(i).in(0).bits.s2Resp := mergeArb(i).out.bits.s2Resp 46192e3bfefSLemover } 46292e3bfefSLemover 46392e3bfefSLemover // io.tlb.map(_.resp) <> outArb.map(_.out) 46492e3bfefSLemover io.tlb.map(_.resp).zip(outArb.map(_.out)).map{ 46592e3bfefSLemover case (resp, out) => resp <> out 46692e3bfefSLemover } 46792e3bfefSLemover 46892e3bfefSLemover // sfence 46992e3bfefSLemover when (flush) { 47092e3bfefSLemover for (i <- 0 until MemReqWidth) { 47192e3bfefSLemover when (waiting_resp(i)) { 47292e3bfefSLemover flush_latch(i) := true.B 47392e3bfefSLemover } 47492e3bfefSLemover } 47592e3bfefSLemover } 47692e3bfefSLemover // mem -> control signal 47792e3bfefSLemover // waiting_resp and sfence_latch will be reset when mem_resp_done 47892e3bfefSLemover when (mem_resp_done) { 47992e3bfefSLemover waiting_resp(mem.d.bits.source) := false.B 48092e3bfefSLemover flush_latch(mem.d.bits.source) := false.B 48192e3bfefSLemover } 48292e3bfefSLemover 48392e3bfefSLemover def block_decoupled[T <: Data](source: DecoupledIO[T], sink: DecoupledIO[T], block_signal: Bool) = { 48492e3bfefSLemover sink.valid := source.valid && !block_signal 48592e3bfefSLemover source.ready := sink.ready && !block_signal 48692e3bfefSLemover sink.bits := source.bits 48792e3bfefSLemover } 48892e3bfefSLemover 48992e3bfefSLemover def get_part(data: Vec[UInt], index: UInt): UInt = { 49092e3bfefSLemover val inner_data = data.asTypeOf(Vec(data.getWidth / XLEN, UInt(XLEN.W))) 49192e3bfefSLemover inner_data(index) 49292e3bfefSLemover } 49392e3bfefSLemover 49463632028SHaoyuan Feng // not_super means that this is a normal page 49563632028SHaoyuan Feng // valididx(i) will be all true when super page to be convenient for l1 tlb matching 49663632028SHaoyuan Feng def contiguous_pte_to_merge_ptwResp(pte: UInt, vpn: UInt, af: Bool, af_first: Boolean, not_super: Boolean = true) : PtwMergeResp = { 49763632028SHaoyuan Feng assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 49863632028SHaoyuan Feng val ptw_merge_resp = Wire(new PtwMergeResp()) 49963632028SHaoyuan Feng for (i <- 0 until tlbcontiguous) { 50063632028SHaoyuan Feng val pte_in = pte(64 * i + 63, 64 * i).asTypeOf(new PteBundle()) 50163632028SHaoyuan Feng val ptw_resp = Wire(new PtwMergeEntry(tagLen = sectorvpnLen, hasPerm = true, hasLevel = true)) 50263632028SHaoyuan Feng ptw_resp.ppn := pte_in.ppn(ppnLen - 1, sectortlbwidth) 50363632028SHaoyuan Feng ptw_resp.ppn_low := pte_in.ppn(sectortlbwidth - 1, 0) 50463632028SHaoyuan Feng ptw_resp.level.map(_ := 2.U) 50563632028SHaoyuan Feng ptw_resp.perm.map(_ := pte_in.getPerm()) 50663632028SHaoyuan Feng ptw_resp.tag := vpn(vpnLen - 1, sectortlbwidth) 50763632028SHaoyuan Feng ptw_resp.pf := (if (af_first) !af else true.B) && pte_in.isPf(2.U) 50863632028SHaoyuan Feng ptw_resp.af := (if (!af_first) pte_in.isPf(2.U) else true.B) && (af || pte_in.isAf()) 50963632028SHaoyuan Feng ptw_resp.v := !ptw_resp.pf 51063632028SHaoyuan Feng ptw_resp.prefetch := DontCare 51163632028SHaoyuan Feng ptw_resp.asid := satp.asid 51263632028SHaoyuan Feng ptw_merge_resp.entry(i) := ptw_resp 51363632028SHaoyuan Feng } 51463632028SHaoyuan Feng ptw_merge_resp.pteidx := UIntToOH(vpn(sectortlbwidth - 1, 0)).asBools 51563632028SHaoyuan Feng ptw_merge_resp.not_super := not_super.B 51663632028SHaoyuan Feng ptw_merge_resp 51763632028SHaoyuan Feng } 51863632028SHaoyuan Feng 51963632028SHaoyuan Feng def merge_ptwResp_to_sector_ptwResp(pte: PtwMergeResp) : PtwSectorResp = { 52063632028SHaoyuan Feng assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 52163632028SHaoyuan Feng val ptw_sector_resp = Wire(new PtwSectorResp) 52263632028SHaoyuan Feng ptw_sector_resp.entry.tag := pte.entry(OHToUInt(pte.pteidx)).tag 52363632028SHaoyuan Feng ptw_sector_resp.entry.asid := pte.entry(OHToUInt(pte.pteidx)).asid 52463632028SHaoyuan Feng ptw_sector_resp.entry.ppn := pte.entry(OHToUInt(pte.pteidx)).ppn 52563632028SHaoyuan Feng ptw_sector_resp.entry.perm.map(_ := pte.entry(OHToUInt(pte.pteidx)).perm.getOrElse(0.U.asTypeOf(new PtePermBundle))) 52663632028SHaoyuan Feng ptw_sector_resp.entry.level.map(_ := pte.entry(OHToUInt(pte.pteidx)).level.getOrElse(0.U(2.W))) 52763632028SHaoyuan Feng ptw_sector_resp.entry.prefetch := pte.entry(OHToUInt(pte.pteidx)).prefetch 52863632028SHaoyuan Feng ptw_sector_resp.entry.v := pte.entry(OHToUInt(pte.pteidx)).v 52963632028SHaoyuan Feng ptw_sector_resp.af := pte.entry(OHToUInt(pte.pteidx)).af 53063632028SHaoyuan Feng ptw_sector_resp.pf := pte.entry(OHToUInt(pte.pteidx)).pf 53163632028SHaoyuan Feng ptw_sector_resp.addr_low := OHToUInt(pte.pteidx) 532b0fa7106SHaoyuan Feng ptw_sector_resp.pteidx := pte.pteidx 53363632028SHaoyuan Feng for (i <- 0 until tlbcontiguous) { 53463632028SHaoyuan Feng val ppn_equal = pte.entry(i).ppn === pte.entry(OHToUInt(pte.pteidx)).ppn 53563632028SHaoyuan Feng val perm_equal = pte.entry(i).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt === pte.entry(OHToUInt(pte.pteidx)).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt 53663632028SHaoyuan Feng val v_equal = pte.entry(i).v === pte.entry(OHToUInt(pte.pteidx)).v 53763632028SHaoyuan Feng val af_equal = pte.entry(i).af === pte.entry(OHToUInt(pte.pteidx)).af 53863632028SHaoyuan Feng val pf_equal = pte.entry(i).pf === pte.entry(OHToUInt(pte.pteidx)).pf 53963632028SHaoyuan Feng ptw_sector_resp.valididx(i) := (ppn_equal && perm_equal && v_equal && af_equal && pf_equal) || !pte.not_super 54063632028SHaoyuan Feng ptw_sector_resp.ppn_low(i) := pte.entry(i).ppn_low 54163632028SHaoyuan Feng } 54263632028SHaoyuan Feng ptw_sector_resp.valididx(OHToUInt(pte.pteidx)) := true.B 54363632028SHaoyuan Feng ptw_sector_resp 54463632028SHaoyuan Feng } 54563632028SHaoyuan Feng 54692e3bfefSLemover def outReady(source: UInt, port: Int): Bool = { 54745f43e6eSTang Haojin MuxLookup(source, true.B)((0 until PtwWidth).map(i => i.U -> mergeArb(i).in(port).ready)) 54892e3bfefSLemover } 54992e3bfefSLemover 55092e3bfefSLemover // debug info 55192e3bfefSLemover for (i <- 0 until PtwWidth) { 55292e3bfefSLemover XSDebug(p"[io.tlb(${i.U})] ${io.tlb(i)}\n") 55392e3bfefSLemover } 5547797f035SbugGenerator XSDebug(p"[sfence] ${io.sfence}\n") 55592e3bfefSLemover XSDebug(p"[io.csr.tlb] ${io.csr.tlb}\n") 55692e3bfefSLemover 55792e3bfefSLemover for (i <- 0 until PtwWidth) { 558935edac4STang Haojin XSPerfAccumulate(s"req_count${i}", io.tlb(i).req(0).fire) 55992e3bfefSLemover XSPerfAccumulate(s"req_blocked_count_${i}", io.tlb(i).req(0).valid && !io.tlb(i).req(0).ready) 56092e3bfefSLemover } 56192e3bfefSLemover XSPerfAccumulate(s"req_blocked_by_mq", arb1.io.out.valid && missQueue.io.out.valid) 56292e3bfefSLemover for (i <- 0 until (MemReqWidth + 1)) { 56392e3bfefSLemover XSPerfAccumulate(s"mem_req_util${i}", PopCount(waiting_resp) === i.U) 56492e3bfefSLemover } 56592e3bfefSLemover XSPerfAccumulate("mem_cycle", PopCount(waiting_resp) =/= 0.U) 566935edac4STang Haojin XSPerfAccumulate("mem_count", mem.a.fire) 567dd7fe201SHaoyuan Feng for (i <- 0 until PtwWidth) { 56863632028SHaoyuan Feng XSPerfAccumulate(s"llptw_ppn_af${i}", mergeArb(i).in(outArbMqPort).valid && mergeArb(i).in(outArbMqPort).bits.entry(OHToUInt(mergeArb(i).in(outArbMqPort).bits.pteidx)).af && !llptw_out.bits.af) 569dd7fe201SHaoyuan Feng XSPerfAccumulate(s"access_fault${i}", io.tlb(i).resp.fire && io.tlb(i).resp.bits.af) 570dd7fe201SHaoyuan Feng } 57192e3bfefSLemover 57292e3bfefSLemover // print configs 573f1fe8698SLemover println(s"${l2tlbParams.name}: a ptw, a llptw with size ${l2tlbParams.llptwsize}, miss queue size ${MissQueueSize} l1:${l2tlbParams.l1Size} fa l2: nSets ${l2tlbParams.l2nSets} nWays ${l2tlbParams.l2nWays} l3: ${l2tlbParams.l3nSets} nWays ${l2tlbParams.l3nWays} blockBytes:${l2tlbParams.blockBytes}") 57492e3bfefSLemover 57592e3bfefSLemover // time out assert 57692e3bfefSLemover for (i <- 0 until MemReqWidth) { 57792e3bfefSLemover TimeOutAssert(waiting_resp(i), timeOutThreshold, s"ptw mem resp time out wait_resp${i}") 57892e3bfefSLemover TimeOutAssert(flush_latch(i), timeOutThreshold, s"ptw mem resp time out flush_latch${i}") 57992e3bfefSLemover } 58092e3bfefSLemover 58192e3bfefSLemover 58292e3bfefSLemover val perfEvents = Seq(llptw, cache, ptw).flatMap(_.getPerfEvents) 58392e3bfefSLemover generatePerfEvent() 5845afdf73cSHaoyuan Feng 585da3bf434SMaxpicca-Li val isWriteL1TlbTable = WireInit(Constantin.createRecord("isWriteL1TlbTable" + p(XSCoreParamsKey).HartId.toString)) 5865afdf73cSHaoyuan Feng val L1TlbTable = ChiselDB.createTable("L1Tlb_hart" + p(XSCoreParamsKey).HartId.toString, new L1TlbDB) 5875afdf73cSHaoyuan Feng val ITlbReqDB, DTlbReqDB, ITlbRespDB, DTlbRespDB = Wire(new L1TlbDB) 5885afdf73cSHaoyuan Feng ITlbReqDB.vpn := io.tlb(0).req(0).bits.vpn 5895afdf73cSHaoyuan Feng DTlbReqDB.vpn := io.tlb(1).req(0).bits.vpn 5905afdf73cSHaoyuan Feng ITlbRespDB.vpn := io.tlb(0).resp.bits.entry.tag 5915afdf73cSHaoyuan Feng DTlbRespDB.vpn := io.tlb(1).resp.bits.entry.tag 592da3bf434SMaxpicca-Li L1TlbTable.log(ITlbReqDB, isWriteL1TlbTable.orR && io.tlb(0).req(0).fire, "ITlbReq", clock, reset) 593da3bf434SMaxpicca-Li L1TlbTable.log(DTlbReqDB, isWriteL1TlbTable.orR && io.tlb(1).req(0).fire, "DTlbReq", clock, reset) 594da3bf434SMaxpicca-Li L1TlbTable.log(ITlbRespDB, isWriteL1TlbTable.orR && io.tlb(0).resp.fire, "ITlbResp", clock, reset) 595da3bf434SMaxpicca-Li L1TlbTable.log(DTlbRespDB, isWriteL1TlbTable.orR && io.tlb(1).resp.fire, "DTlbResp", clock, reset) 5965afdf73cSHaoyuan Feng 597da3bf434SMaxpicca-Li val isWritePageCacheTable = WireInit(Constantin.createRecord("isWritePageCacheTable" + p(XSCoreParamsKey).HartId.toString)) 5985afdf73cSHaoyuan Feng val PageCacheTable = ChiselDB.createTable("PageCache_hart" + p(XSCoreParamsKey).HartId.toString, new PageCacheDB) 5995afdf73cSHaoyuan Feng val PageCacheDB = Wire(new PageCacheDB) 60063632028SHaoyuan Feng PageCacheDB.vpn := Cat(cache.io.resp.bits.toTlb.entry(0).tag, OHToUInt(cache.io.resp.bits.toTlb.pteidx)) 6015afdf73cSHaoyuan Feng PageCacheDB.source := cache.io.resp.bits.req_info.source 6025afdf73cSHaoyuan Feng PageCacheDB.bypassed := cache.io.resp.bits.bypassed 6035afdf73cSHaoyuan Feng PageCacheDB.is_first := cache.io.resp.bits.isFirst 60463632028SHaoyuan Feng PageCacheDB.prefetched := cache.io.resp.bits.toTlb.entry(0).prefetch 6055afdf73cSHaoyuan Feng PageCacheDB.prefetch := cache.io.resp.bits.prefetch 6065afdf73cSHaoyuan Feng PageCacheDB.l2Hit := cache.io.resp.bits.toFsm.l2Hit 6075afdf73cSHaoyuan Feng PageCacheDB.l1Hit := cache.io.resp.bits.toFsm.l1Hit 6085afdf73cSHaoyuan Feng PageCacheDB.hit := cache.io.resp.bits.hit 609da3bf434SMaxpicca-Li PageCacheTable.log(PageCacheDB, isWritePageCacheTable.orR && cache.io.resp.fire, "PageCache", clock, reset) 6105afdf73cSHaoyuan Feng 611da3bf434SMaxpicca-Li val isWritePTWTable = WireInit(Constantin.createRecord("isWritePTWTable" + p(XSCoreParamsKey).HartId.toString)) 6125afdf73cSHaoyuan Feng val PTWTable = ChiselDB.createTable("PTW_hart" + p(XSCoreParamsKey).HartId.toString, new PTWDB) 6135afdf73cSHaoyuan Feng val PTWReqDB, PTWRespDB, LLPTWReqDB, LLPTWRespDB = Wire(new PTWDB) 6145afdf73cSHaoyuan Feng PTWReqDB.vpn := ptw.io.req.bits.req_info.vpn 6155afdf73cSHaoyuan Feng PTWReqDB.source := ptw.io.req.bits.req_info.source 6165afdf73cSHaoyuan Feng PTWRespDB.vpn := ptw.io.refill.req_info.vpn 6175afdf73cSHaoyuan Feng PTWRespDB.source := ptw.io.refill.req_info.source 6185afdf73cSHaoyuan Feng LLPTWReqDB.vpn := llptw.io.in.bits.req_info.vpn 6195afdf73cSHaoyuan Feng LLPTWReqDB.source := llptw.io.in.bits.req_info.source 6205afdf73cSHaoyuan Feng LLPTWRespDB.vpn := llptw.io.mem.refill.vpn 6215afdf73cSHaoyuan Feng LLPTWRespDB.source := llptw.io.mem.refill.source 622da3bf434SMaxpicca-Li PTWTable.log(PTWReqDB, isWritePTWTable.orR && ptw.io.req.fire, "PTWReq", clock, reset) 623da3bf434SMaxpicca-Li PTWTable.log(PTWRespDB, isWritePTWTable.orR && ptw.io.mem.resp.fire, "PTWResp", clock, reset) 624da3bf434SMaxpicca-Li PTWTable.log(LLPTWReqDB, isWritePTWTable.orR && llptw.io.in.fire, "LLPTWReq", clock, reset) 625da3bf434SMaxpicca-Li PTWTable.log(LLPTWRespDB, isWritePTWTable.orR && llptw.io.mem.resp.fire, "LLPTWResp", clock, reset) 6265afdf73cSHaoyuan Feng 627da3bf434SMaxpicca-Li val isWriteL2TlbMissQueueTable = WireInit(Constantin.createRecord("isWriteL2TlbMissQueueTable" + p(XSCoreParamsKey).HartId.toString)) 6285afdf73cSHaoyuan Feng val L2TlbMissQueueTable = ChiselDB.createTable("L2TlbMissQueue_hart" + p(XSCoreParamsKey).HartId.toString, new L2TlbMissQueueDB) 6295afdf73cSHaoyuan Feng val L2TlbMissQueueInDB, L2TlbMissQueueOutDB = Wire(new L2TlbMissQueueDB) 6305afdf73cSHaoyuan Feng L2TlbMissQueueInDB.vpn := missQueue.io.in.bits.vpn 6315afdf73cSHaoyuan Feng L2TlbMissQueueOutDB.vpn := missQueue.io.out.bits.vpn 632da3bf434SMaxpicca-Li L2TlbMissQueueTable.log(L2TlbMissQueueInDB, isWriteL2TlbMissQueueTable.orR && missQueue.io.in.fire, "L2TlbMissQueueIn", clock, reset) 633da3bf434SMaxpicca-Li L2TlbMissQueueTable.log(L2TlbMissQueueOutDB, isWriteL2TlbMissQueueTable.orR && missQueue.io.out.fire, "L2TlbMissQueueOut", clock, reset) 63492e3bfefSLemover} 63592e3bfefSLemover 6367797f035SbugGenerator/** BlockHelper, block missqueue, not to send too many req to cache 6377797f035SbugGenerator * Parameter: 6387797f035SbugGenerator * enable: enable BlockHelper, mq should not send too many reqs 6397797f035SbugGenerator * start: when miss queue out fire and need, block miss queue's out 6407797f035SbugGenerator * block: block miss queue's out 6417797f035SbugGenerator * latency: last missqueue out's cache access latency 6427797f035SbugGenerator */ 6437797f035SbugGeneratorclass BlockHelper(latency: Int)(implicit p: Parameters) extends XSModule { 6447797f035SbugGenerator val io = IO(new Bundle { 6457797f035SbugGenerator val enable = Input(Bool()) 6467797f035SbugGenerator val start = Input(Bool()) 6477797f035SbugGenerator val block = Output(Bool()) 6487797f035SbugGenerator }) 6497797f035SbugGenerator 6507797f035SbugGenerator val count = RegInit(0.U(log2Ceil(latency).W)) 6517797f035SbugGenerator val valid = RegInit(false.B) 6527797f035SbugGenerator val work = RegInit(true.B) 6537797f035SbugGenerator 6547797f035SbugGenerator io.block := valid 6557797f035SbugGenerator 6567797f035SbugGenerator when (io.start && work) { valid := true.B } 6577797f035SbugGenerator when (valid) { count := count + 1.U } 6587797f035SbugGenerator when (count === (latency.U) || io.enable) { 6597797f035SbugGenerator valid := false.B 6607797f035SbugGenerator work := io.enable 6617797f035SbugGenerator count := 0.U 6627797f035SbugGenerator } 6637797f035SbugGenerator} 6647797f035SbugGenerator 66592e3bfefSLemoverclass PTEHelper() extends ExtModule { 66692e3bfefSLemover val clock = IO(Input(Clock())) 66792e3bfefSLemover val enable = IO(Input(Bool())) 66892e3bfefSLemover val satp = IO(Input(UInt(64.W))) 66992e3bfefSLemover val vpn = IO(Input(UInt(64.W))) 67092e3bfefSLemover val pte = IO(Output(UInt(64.W))) 67192e3bfefSLemover val level = IO(Output(UInt(8.W))) 67292e3bfefSLemover val pf = IO(Output(UInt(8.W))) 67392e3bfefSLemover} 67492e3bfefSLemover 6755afdf73cSHaoyuan Fengclass PTWDelayN[T <: Data](gen: T, n: Int, flush: Bool) extends Module { 6765afdf73cSHaoyuan Feng val io = IO(new Bundle() { 6775afdf73cSHaoyuan Feng val in = Input(gen) 6785afdf73cSHaoyuan Feng val out = Output(gen) 6795afdf73cSHaoyuan Feng val ptwflush = Input(flush.cloneType) 6805afdf73cSHaoyuan Feng }) 6815afdf73cSHaoyuan Feng val out = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(gen)))) 6825afdf73cSHaoyuan Feng val t = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(gen)))) 6835afdf73cSHaoyuan Feng out(0) := io.in 6845afdf73cSHaoyuan Feng if (n == 1) { 6855afdf73cSHaoyuan Feng io.out := out(0) 6865afdf73cSHaoyuan Feng } else { 6875afdf73cSHaoyuan Feng when (io.ptwflush) { 6885afdf73cSHaoyuan Feng for (i <- 0 until n) { 6895afdf73cSHaoyuan Feng t(i) := 0.U.asTypeOf(gen) 6905afdf73cSHaoyuan Feng out(i) := 0.U.asTypeOf(gen) 6915afdf73cSHaoyuan Feng } 6925afdf73cSHaoyuan Feng io.out := 0.U.asTypeOf(gen) 6935afdf73cSHaoyuan Feng } .otherwise { 6945afdf73cSHaoyuan Feng for (i <- 1 until n) { 6955afdf73cSHaoyuan Feng t(i-1) := out(i-1) 6965afdf73cSHaoyuan Feng out(i) := t(i-1) 6975afdf73cSHaoyuan Feng } 6985afdf73cSHaoyuan Feng io.out := out(n-1) 6995afdf73cSHaoyuan Feng } 7005afdf73cSHaoyuan Feng } 7015afdf73cSHaoyuan Feng} 7025afdf73cSHaoyuan Feng 7035afdf73cSHaoyuan Fengobject PTWDelayN { 7045afdf73cSHaoyuan Feng def apply[T <: Data](in: T, n: Int, flush: Bool): T = { 7055afdf73cSHaoyuan Feng val delay = Module(new PTWDelayN(in.cloneType, n, flush)) 7065afdf73cSHaoyuan Feng delay.io.in := in 7075afdf73cSHaoyuan Feng delay.io.ptwflush := flush 7085afdf73cSHaoyuan Feng delay.io.out 7095afdf73cSHaoyuan Feng } 7105afdf73cSHaoyuan Feng} 7115afdf73cSHaoyuan Feng 71292e3bfefSLemoverclass FakePTW()(implicit p: Parameters) extends XSModule with HasPtwConst { 71392e3bfefSLemover val io = IO(new L2TLBIO) 7145afdf73cSHaoyuan Feng val flush = VecInit(Seq.fill(PtwWidth)(false.B)) 7155afdf73cSHaoyuan Feng flush(0) := DelayN(io.sfence.valid || io.csr.tlb.satp.changed, itlbParams.fenceDelay) 7165afdf73cSHaoyuan Feng flush(1) := DelayN(io.sfence.valid || io.csr.tlb.satp.changed, ldtlbParams.fenceDelay) 71792e3bfefSLemover for (i <- 0 until PtwWidth) { 71892e3bfefSLemover val helper = Module(new PTEHelper()) 71992e3bfefSLemover helper.clock := clock 72092e3bfefSLemover helper.satp := io.csr.tlb.satp.ppn 7215afdf73cSHaoyuan Feng 7225afdf73cSHaoyuan Feng if (coreParams.softPTWDelay == 1) { 7235afdf73cSHaoyuan Feng helper.enable := io.tlb(i).req(0).fire 72492e3bfefSLemover helper.vpn := io.tlb(i).req(0).bits.vpn 7255afdf73cSHaoyuan Feng } else { 7265afdf73cSHaoyuan Feng helper.enable := PTWDelayN(io.tlb(i).req(0).fire, coreParams.softPTWDelay - 1, flush(i)) 7275afdf73cSHaoyuan Feng helper.vpn := PTWDelayN(io.tlb(i).req(0).bits.vpn, coreParams.softPTWDelay - 1, flush(i)) 7285afdf73cSHaoyuan Feng } 7295afdf73cSHaoyuan Feng 73092e3bfefSLemover val pte = helper.pte.asTypeOf(new PteBundle) 73192e3bfefSLemover val level = helper.level 73292e3bfefSLemover val pf = helper.pf 7335afdf73cSHaoyuan Feng val empty = RegInit(true.B) 7345afdf73cSHaoyuan Feng when (io.tlb(i).req(0).fire) { 7355afdf73cSHaoyuan Feng empty := false.B 7365afdf73cSHaoyuan Feng } .elsewhen (io.tlb(i).resp.fire || flush(i)) { 7375afdf73cSHaoyuan Feng empty := true.B 7385afdf73cSHaoyuan Feng } 73992e3bfefSLemover 7405afdf73cSHaoyuan Feng io.tlb(i).req(0).ready := empty || io.tlb(i).resp.fire 7415afdf73cSHaoyuan Feng io.tlb(i).resp.valid := PTWDelayN(io.tlb(i).req(0).fire, coreParams.softPTWDelay, flush(i)) 74292e3bfefSLemover assert(!io.tlb(i).resp.valid || io.tlb(i).resp.ready) 7435afdf73cSHaoyuan Feng io.tlb(i).resp.bits.entry.tag := PTWDelayN(io.tlb(i).req(0).bits.vpn, coreParams.softPTWDelay, flush(i)) 74492e3bfefSLemover io.tlb(i).resp.bits.entry.ppn := pte.ppn 74592e3bfefSLemover io.tlb(i).resp.bits.entry.perm.map(_ := pte.getPerm()) 74692e3bfefSLemover io.tlb(i).resp.bits.entry.level.map(_ := level) 74792e3bfefSLemover io.tlb(i).resp.bits.pf := pf 74892e3bfefSLemover io.tlb(i).resp.bits.af := DontCare // TODO: implement it 7495afdf73cSHaoyuan Feng io.tlb(i).resp.bits.entry.v := !pf 7505afdf73cSHaoyuan Feng io.tlb(i).resp.bits.entry.prefetch := DontCare 7515afdf73cSHaoyuan Feng io.tlb(i).resp.bits.entry.asid := io.csr.tlb.satp.asid 75292e3bfefSLemover } 75392e3bfefSLemover} 75492e3bfefSLemover 75592e3bfefSLemoverclass L2TLBWrapper()(implicit p: Parameters) extends LazyModule with HasXSParameter { 75695e60e55STang Haojin override def shouldBeInlined: Boolean = false 75792e3bfefSLemover val useSoftPTW = coreParams.softPTW 75892e3bfefSLemover val node = if (!useSoftPTW) TLIdentityNode() else null 75992e3bfefSLemover val ptw = if (!useSoftPTW) LazyModule(new L2TLB()) else null 76092e3bfefSLemover if (!useSoftPTW) { 76192e3bfefSLemover node := ptw.node 76292e3bfefSLemover } 76392e3bfefSLemover 764935edac4STang Haojin class L2TLBWrapperImp(wrapper: LazyModule) extends LazyModuleImp(wrapper) with HasPerfEvents { 76592e3bfefSLemover val io = IO(new L2TLBIO) 76692e3bfefSLemover val perfEvents = if (useSoftPTW) { 76792e3bfefSLemover val fake_ptw = Module(new FakePTW()) 76892e3bfefSLemover io <> fake_ptw.io 76992e3bfefSLemover Seq() 77092e3bfefSLemover } 77192e3bfefSLemover else { 77292e3bfefSLemover io <> ptw.module.io 77392e3bfefSLemover ptw.module.getPerfEvents 77492e3bfefSLemover } 77592e3bfefSLemover generatePerfEvent() 77692e3bfefSLemover } 777935edac4STang Haojin 778935edac4STang Haojin lazy val module = new L2TLBWrapperImp(this) 77992e3bfefSLemover} 780