xref: /XiangShan/src/main/scala/xiangshan/cache/mmu/L2TLB.scala (revision 82978df9e4a523e2f864bf37c6fb8e793e64f59a)
192e3bfefSLemover/***************************************************************************************
292e3bfefSLemover* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
392e3bfefSLemover* Copyright (c) 2020-2021 Peng Cheng Laboratory
492e3bfefSLemover*
592e3bfefSLemover* XiangShan is licensed under Mulan PSL v2.
692e3bfefSLemover* You can use this software according to the terms and conditions of the Mulan PSL v2.
792e3bfefSLemover* You may obtain a copy of Mulan PSL v2 at:
892e3bfefSLemover*          http://license.coscl.org.cn/MulanPSL2
992e3bfefSLemover*
1092e3bfefSLemover* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
1192e3bfefSLemover* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
1292e3bfefSLemover* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
1392e3bfefSLemover*
1492e3bfefSLemover* See the Mulan PSL v2 for more details.
1592e3bfefSLemover***************************************************************************************/
1692e3bfefSLemover
1792e3bfefSLemoverpackage xiangshan.cache.mmu
1892e3bfefSLemover
198891a219SYinan Xuimport org.chipsalliance.cde.config.Parameters
2092e3bfefSLemoverimport chisel3._
2192e3bfefSLemoverimport chisel3.experimental.ExtModule
2292e3bfefSLemoverimport chisel3.util._
2392e3bfefSLemoverimport xiangshan._
2492e3bfefSLemoverimport xiangshan.cache.{HasDCacheParameters, MemoryOpConstants}
2592e3bfefSLemoverimport utils._
263c02ee8fSwakafaimport utility._
2792e3bfefSLemoverimport freechips.rocketchip.diplomacy.{IdRange, LazyModule, LazyModuleImp}
2892e3bfefSLemoverimport freechips.rocketchip.tilelink._
2992e3bfefSLemoverimport xiangshan.backend.fu.{PMP, PMPChecker, PMPReqBundle, PMPRespBundle}
3092e3bfefSLemoverimport xiangshan.backend.fu.util.HasCSRConst
319c26bab7SHaoyuan Fengimport difftest._
3292e3bfefSLemover
3392e3bfefSLemoverclass L2TLB()(implicit p: Parameters) extends LazyModule with HasPtwConst {
3495e60e55STang Haojin  override def shouldBeInlined: Boolean = false
3592e3bfefSLemover
3692e3bfefSLemover  val node = TLClientNode(Seq(TLMasterPortParameters.v1(
3792e3bfefSLemover    clients = Seq(TLMasterParameters.v1(
3892e3bfefSLemover      "ptw",
3992e3bfefSLemover      sourceId = IdRange(0, MemReqWidth)
40d2b20d1aSTang Haojin    )),
41d2b20d1aSTang Haojin    requestFields = Seq(ReqSourceField())
4292e3bfefSLemover  )))
4392e3bfefSLemover
4492e3bfefSLemover  lazy val module = new L2TLBImp(this)
4592e3bfefSLemover}
4692e3bfefSLemover
4792e3bfefSLemoverclass L2TLBImp(outer: L2TLB)(implicit p: Parameters) extends PtwModule(outer) with HasCSRConst with HasPerfEvents {
4892e3bfefSLemover
4992e3bfefSLemover  val (mem, edge) = outer.node.out.head
5092e3bfefSLemover
5192e3bfefSLemover  val io = IO(new L2TLBIO)
5292e3bfefSLemover  val difftestIO = IO(new Bundle() {
5392e3bfefSLemover    val ptwResp = Output(Bool())
5492e3bfefSLemover    val ptwAddr = Output(UInt(64.W))
5592e3bfefSLemover    val ptwData = Output(Vec(4, UInt(64.W)))
5692e3bfefSLemover  })
5792e3bfefSLemover
5892e3bfefSLemover  /* Ptw processes multiple requests
5992e3bfefSLemover   * Divide Ptw procedure into two stages: cache access ; mem access if cache miss
6092e3bfefSLemover   *           miss queue itlb       dtlb
6192e3bfefSLemover   *               |       |         |
6292e3bfefSLemover   *               ------arbiter------
6392e3bfefSLemover   *                            |
6492e3bfefSLemover   *                    l1 - l2 - l3 - sp
6592e3bfefSLemover   *                            |
6692e3bfefSLemover   *          -------------------------------------------
6792e3bfefSLemover   *    miss  |  queue                                  | hit
6892e3bfefSLemover   *    [][][][][][]                                    |
6992e3bfefSLemover   *          |                                         |
7092e3bfefSLemover   *    state machine accessing mem                     |
7192e3bfefSLemover   *          |                                         |
7292e3bfefSLemover   *          ---------------arbiter---------------------
7392e3bfefSLemover   *                 |                    |
7492e3bfefSLemover   *                itlb                 dtlb
7592e3bfefSLemover   */
7692e3bfefSLemover
7792e3bfefSLemover  difftestIO <> DontCare
7892e3bfefSLemover
797797f035SbugGenerator  val sfence_tmp = DelayN(io.sfence, 1)
807797f035SbugGenerator  val csr_tmp    = DelayN(io.csr.tlb, 1)
81d0de7e4aSpeixiaokun  val sfence_dup = Seq.fill(9)(RegNext(sfence_tmp))
82d0de7e4aSpeixiaokun  val csr_dup = Seq.fill(8)(RegNext(csr_tmp))
837797f035SbugGenerator  val satp   = csr_dup(0).satp
84d0de7e4aSpeixiaokun  val vsatp  = csr_dup(0).vsatp
85d0de7e4aSpeixiaokun  val hgatp  = csr_dup(0).hgatp
867797f035SbugGenerator  val priv   = csr_dup(0).priv
87d0de7e4aSpeixiaokun  val flush  = sfence_dup(0).valid || satp.changed || vsatp.changed || hgatp.changed
8892e3bfefSLemover
8992e3bfefSLemover  val pmp = Module(new PMP())
9092e3bfefSLemover  val pmp_check = VecInit(Seq.fill(2)(Module(new PMPChecker(lgMaxSize = 3, sameCycle = true)).io))
9192e3bfefSLemover  pmp.io.distribute_csr := io.csr.distribute_csr
9292e3bfefSLemover  pmp_check.foreach(_.check_env.apply(ModeS, pmp.io.pmp, pmp.io.pma))
9392e3bfefSLemover
9492e3bfefSLemover  val missQueue = Module(new L2TlbMissQueue)
9592e3bfefSLemover  val cache = Module(new PtwCache)
9692e3bfefSLemover  val ptw = Module(new PTW)
97d0de7e4aSpeixiaokun  val hptw = Module(new HPTW)
9892e3bfefSLemover  val llptw = Module(new LLPTW)
997797f035SbugGenerator  val blockmq = Module(new BlockHelper(3))
10092e3bfefSLemover  val arb1 = Module(new Arbiter(new PtwReq, PtwWidth))
10192e3bfefSLemover  val arb2 = Module(new Arbiter(new Bundle {
10292e3bfefSLemover    val vpn = UInt(vpnLen.W)
103d0de7e4aSpeixiaokun    val s2xlate = UInt(2.W)
10492e3bfefSLemover    val source = UInt(bSourceWidth.W)
105d0de7e4aSpeixiaokun  }, if (l2tlbParams.enablePrefetch) 4 else 3 + if(l2tlbParams.HasHExtension) 1 else 0))
106d0de7e4aSpeixiaokun  val hptw_req_arb = Module(new Arbiter(new Bundle {
107d0de7e4aSpeixiaokun    val id = UInt(log2Up(l2tlbParams.llptwsize).W)
108*82978df9Speixiaokun    val gvpn = UInt(vpnLen.W)
109d0de7e4aSpeixiaokun  }, 2))
110d0de7e4aSpeixiaokun  val hptw_resp_arb = Module(new Arbiter(new Bundle {
111d0de7e4aSpeixiaokun    val resp = new HptwResp()
112d0de7e4aSpeixiaokun    val id = UInt(log2Up(l2tlbParams.llptwsize).W)
113d0de7e4aSpeixiaokun  }, 2))
114d0de7e4aSpeixiaokun  val outArb = (0 until PtwWidth).map(i => Module(new Arbiter(new Bundle {
115d0de7e4aSpeixiaokun    val s2xlate = UInt(2.W)
116d0de7e4aSpeixiaokun    val s1Resp = new PtwSectorResp ()
117d0de7e4aSpeixiaokun    val s2Resp = new HptwResp()
118d0de7e4aSpeixiaokun  }, 1)).io)
119d0de7e4aSpeixiaokun  val mergeArb = (0 until PtwWidth).map(i => Module(new Arbiter(new Bundle {
120d0de7e4aSpeixiaokun    val s2xlate = UInt(2.W)
121d0de7e4aSpeixiaokun    val s1Resp = new PtwMergeResp()
122d0de7e4aSpeixiaokun    val s2Resp = new HptwResp()
123d0de7e4aSpeixiaokun  }, 3)).io)
12492e3bfefSLemover  val outArbCachePort = 0
12592e3bfefSLemover  val outArbFsmPort = 1
12692e3bfefSLemover  val outArbMqPort = 2
12792e3bfefSLemover
128d0de7e4aSpeixiaokun  // hptw arb input port
129d0de7e4aSpeixiaokun  val InHptwArbPTWPort = 0
130d0de7e4aSpeixiaokun  val InHptwArbLLPTWPort = 1
131d0de7e4aSpeixiaokun  hptw_req_arb.io.in(InHptwArbPTWPort).valid := ptw.io.hptw.req.valid
132d0de7e4aSpeixiaokun  hptw_req_arb.io.in(InHptwArbPTWPort).bits.gvpn := ptw.io.hptw.req.bits.gvpn
133d0de7e4aSpeixiaokun  hptw_req_arb.io.in(InHptwArbPTWPort).bits.id := ptw.io.hptw.req.bits.id
134d0de7e4aSpeixiaokun  ptw.io.hptw.req.ready := hptw_req_arb.io.in(InHptwArbPTWPort).ready
135d0de7e4aSpeixiaokun
136d0de7e4aSpeixiaokun  hptw_req_arb.io.in(InHptwArbLLPTWPort).valid := llptw.io.hptw.req.valid
137d0de7e4aSpeixiaokun  hptw_req_arb.io.in(InHptwArbLLPTWPort).bits.gvpn := llptw.io.hptw.req.bits.gvpn
138d0de7e4aSpeixiaokun  hptw_req_arb.io.in(InHptwArbLLPTWPort).bits.id := llptw.io.hptw.req.bits.id
139d0de7e4aSpeixiaokun  llptw.io.hptw.req.ready := hptw_req_arb.io.in(InHptwArbLLPTWPort).ready
140d0de7e4aSpeixiaokun
141d0de7e4aSpeixiaokun  val hptw_id = RegInit(0.U(log2Up(l2tlbParams.llptwsize).W))
142d0de7e4aSpeixiaokun  when(hptw_req_arb.io.out.valid) {
143d0de7e4aSpeixiaokun    hptw_id := hptw_req_arb.io.out.bits.id
144d0de7e4aSpeixiaokun  }
1459c503409SLemover  // arb2 input port
1469c503409SLemover  val InArbPTWPort = 0
1479c503409SLemover  val InArbMissQueuePort = 1
1489c503409SLemover  val InArbTlbPort = 2
1499c503409SLemover  val InArbPrefetchPort = 3
150d0de7e4aSpeixiaokun  val InArbHPTWPort = 4
15192e3bfefSLemover  // NOTE: when cache out but miss and ptw doesnt accept,
15292e3bfefSLemover  arb1.io.in <> VecInit(io.tlb.map(_.req(0)))
1539c503409SLemover  arb1.io.out.ready := arb2.io.in(InArbTlbPort).ready
15492e3bfefSLemover
1559c503409SLemover  arb2.io.in(InArbPTWPort).valid := ptw.io.llptw.valid
1569c503409SLemover  arb2.io.in(InArbPTWPort).bits.vpn := ptw.io.llptw.bits.req_info.vpn
157d0de7e4aSpeixiaokun  arb2.io.in(InArbPTWPort).bits.s2xlate := ptw.io.llptw.bits.req_info.s2xlate
1589c503409SLemover  arb2.io.in(InArbPTWPort).bits.source := ptw.io.llptw.bits.req_info.source
1599c503409SLemover  ptw.io.llptw.ready := arb2.io.in(InArbPTWPort).ready
16092e3bfefSLemover  block_decoupled(missQueue.io.out, arb2.io.in(InArbMissQueuePort), !ptw.io.req.ready)
1617797f035SbugGenerator
16292e3bfefSLemover  arb2.io.in(InArbTlbPort).valid := arb1.io.out.valid
16392e3bfefSLemover  arb2.io.in(InArbTlbPort).bits.vpn := arb1.io.out.bits.vpn
164d0de7e4aSpeixiaokun  arb2.io.in(InArbTlbPort).bits.s2xlate := arb1.io.out.bits.s2xlate
16592e3bfefSLemover  arb2.io.in(InArbTlbPort).bits.source := arb1.io.chosen
166d0de7e4aSpeixiaokun
167d0de7e4aSpeixiaokun  arb2.io.in(InArbHPTWPort).valid := hptw_req_arb.io.out.valid
168*82978df9Speixiaokun  arb2.io.in(InArbHPTWPort).bits.vpn := hptw_req_arb.io.out.bits.gvpn
169*82978df9Speixiaokun  arb2.io.in(InArbHPTWPort).bits.s2xlate := onlyStage2
170d0de7e4aSpeixiaokun  arb2.io.in(InArbHPTWPort).bits.source := DontCare
17192e3bfefSLemover  if (l2tlbParams.enablePrefetch) {
17292e3bfefSLemover    val prefetch = Module(new L2TlbPrefetch())
17392e3bfefSLemover    val recv = cache.io.resp
17492e3bfefSLemover    // NOTE: 1. prefetch doesn't gen prefetch 2. req from mq doesn't gen prefetch
17592e3bfefSLemover    // NOTE: 1. miss req gen prefetch 2. hit but prefetched gen prefetch
176935edac4STang Haojin    prefetch.io.in.valid := recv.fire && !from_pre(recv.bits.req_info.source) && (!recv.bits.hit  ||
17792e3bfefSLemover      recv.bits.prefetch) && recv.bits.isFirst
17892e3bfefSLemover    prefetch.io.in.bits.vpn := recv.bits.req_info.vpn
1797797f035SbugGenerator    prefetch.io.sfence := sfence_dup(0)
1807797f035SbugGenerator    prefetch.io.csr := csr_dup(0)
18192e3bfefSLemover    arb2.io.in(InArbPrefetchPort) <> prefetch.io.out
1825afdf73cSHaoyuan Feng
183da3bf434SMaxpicca-Li    val isWriteL2TlbPrefetchTable = WireInit(Constantin.createRecord("isWriteL2TlbPrefetchTable" + p(XSCoreParamsKey).HartId.toString))
1845afdf73cSHaoyuan Feng    val L2TlbPrefetchTable = ChiselDB.createTable("L2TlbPrefetch_hart" + p(XSCoreParamsKey).HartId.toString, new L2TlbPrefetchDB)
1855afdf73cSHaoyuan Feng    val L2TlbPrefetchDB = Wire(new L2TlbPrefetchDB)
1865afdf73cSHaoyuan Feng    L2TlbPrefetchDB.vpn := prefetch.io.out.bits.vpn
187da3bf434SMaxpicca-Li    L2TlbPrefetchTable.log(L2TlbPrefetchDB, isWriteL2TlbPrefetchTable.orR && prefetch.io.out.fire, "L2TlbPrefetch", clock, reset)
18892e3bfefSLemover  }
18992e3bfefSLemover  arb2.io.out.ready := cache.io.req.ready
19092e3bfefSLemover
1917797f035SbugGenerator
1927797f035SbugGenerator  val mq_arb = Module(new Arbiter(new L2TlbInnerBundle, 2))
1937797f035SbugGenerator  mq_arb.io.in(0).valid := cache.io.resp.valid && !cache.io.resp.bits.hit &&
1947797f035SbugGenerator    (!cache.io.resp.bits.toFsm.l2Hit || cache.io.resp.bits.bypassed) &&
1957797f035SbugGenerator    !from_pre(cache.io.resp.bits.req_info.source) &&
1967797f035SbugGenerator    (cache.io.resp.bits.bypassed || cache.io.resp.bits.isFirst || !ptw.io.req.ready)
1977797f035SbugGenerator  mq_arb.io.in(0).bits :=  cache.io.resp.bits.req_info
1987797f035SbugGenerator  mq_arb.io.in(1) <> llptw.io.cache
1997797f035SbugGenerator  missQueue.io.in <> mq_arb.io.out
2007797f035SbugGenerator  missQueue.io.sfence  := sfence_dup(6)
2017797f035SbugGenerator  missQueue.io.csr := csr_dup(5)
2027797f035SbugGenerator
2037797f035SbugGenerator  blockmq.io.start := missQueue.io.out.fire
204935edac4STang Haojin  blockmq.io.enable := ptw.io.req.fire
2057797f035SbugGenerator
206d0de7e4aSpeixiaokun  llptw.io.in.valid := cache.io.resp.valid &&
207d0de7e4aSpeixiaokun    !cache.io.resp.bits.hit &&
208d0de7e4aSpeixiaokun    cache.io.resp.bits.toFsm.l2Hit &&
209d0de7e4aSpeixiaokun    !cache.io.resp.bits.bypassed &&
210d0de7e4aSpeixiaokun    !cache.io.resp.bits.isHptw
2119c503409SLemover  llptw.io.in.bits.req_info := cache.io.resp.bits.req_info
2129c503409SLemover  llptw.io.in.bits.ppn := cache.io.resp.bits.toFsm.ppn
2137797f035SbugGenerator  llptw.io.sfence := sfence_dup(1)
2147797f035SbugGenerator  llptw.io.csr := csr_dup(1)
21592e3bfefSLemover
21692e3bfefSLemover  cache.io.req.valid := arb2.io.out.valid
21792e3bfefSLemover  cache.io.req.bits.req_info.vpn := arb2.io.out.bits.vpn
218d0de7e4aSpeixiaokun  cache.io.req.bits.req_info.s2xlate := arb2.io.out.bits.s2xlate
21992e3bfefSLemover  cache.io.req.bits.req_info.source := arb2.io.out.bits.source
22092e3bfefSLemover  cache.io.req.bits.isFirst := arb2.io.chosen =/= InArbMissQueuePort.U
221d0de7e4aSpeixiaokun  cache.io.req.bits.isHptw := arb2.io.chosen === InArbHPTWPort.U
222d0de7e4aSpeixiaokun  cache.io.req.bits.hptwId := hptw_id
2231f4a7c0cSLemover  cache.io.req.bits.bypassed.map(_ := false.B)
2247797f035SbugGenerator  cache.io.sfence := sfence_dup(2)
2257797f035SbugGenerator  cache.io.csr := csr_dup(2)
2267797f035SbugGenerator  cache.io.sfence_dup.zip(sfence_dup.drop(2).take(4)).map(s => s._1 := s._2)
2277797f035SbugGenerator  cache.io.csr_dup.zip(csr_dup.drop(2).take(3)).map(c => c._1 := c._2)
22892e3bfefSLemover  cache.io.resp.ready := Mux(cache.io.resp.bits.hit,
22992e3bfefSLemover    outReady(cache.io.resp.bits.req_info.source, outArbCachePort),
2309c503409SLemover    Mux(cache.io.resp.bits.toFsm.l2Hit && !cache.io.resp.bits.bypassed, llptw.io.in.ready,
2317797f035SbugGenerator    Mux(cache.io.resp.bits.bypassed || cache.io.resp.bits.isFirst, mq_arb.io.in(0).ready, mq_arb.io.in(0).ready || ptw.io.req.ready)))
23292e3bfefSLemover
23392e3bfefSLemover  // NOTE: missQueue req has higher priority
2347797f035SbugGenerator  ptw.io.req.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && !cache.io.resp.bits.toFsm.l2Hit &&
2357797f035SbugGenerator    !cache.io.resp.bits.bypassed &&
236d0de7e4aSpeixiaokun    !cache.io.resp.bits.isFirst &&
237d0de7e4aSpeixiaokun    !cache.io.resp.bits.isHptw
23892e3bfefSLemover  ptw.io.req.bits.req_info := cache.io.resp.bits.req_info
23992e3bfefSLemover  ptw.io.req.bits.l1Hit := cache.io.resp.bits.toFsm.l1Hit
24092e3bfefSLemover  ptw.io.req.bits.ppn := cache.io.resp.bits.toFsm.ppn
2417797f035SbugGenerator  ptw.io.sfence := sfence_dup(7)
2427797f035SbugGenerator  ptw.io.csr := csr_dup(6)
24392e3bfefSLemover  ptw.io.resp.ready := outReady(ptw.io.resp.bits.source, outArbFsmPort)
24492e3bfefSLemover
245d0de7e4aSpeixiaokun  hptw.io.req.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && !cache.io.resp.bits.bypassed & cache.io.resp.bits.isHptw
246*82978df9Speixiaokun  hptw.io.req.bits.gvpn := cache.io.resp.bits.req_info.vpn
247d0de7e4aSpeixiaokun  hptw.io.req.bits.id := cache.io.resp.bits.toHptw.id
248d0de7e4aSpeixiaokun  hptw.io.req.bits.l1Hit := cache.io.resp.bits.toHptw.l1Hit
249d0de7e4aSpeixiaokun  hptw.io.req.bits.l2Hit := cache.io.resp.bits.toHptw.l2Hit
250d0de7e4aSpeixiaokun  hptw.io.sfence := sfence_dup(8)
251d0de7e4aSpeixiaokun  hptw.io.csr := csr_dup(7)
25292e3bfefSLemover  // mem req
25392e3bfefSLemover  def blockBytes_align(addr: UInt) = {
25492e3bfefSLemover    Cat(addr(PAddrBits - 1, log2Up(l2tlbParams.blockBytes)), 0.U(log2Up(l2tlbParams.blockBytes).W))
25592e3bfefSLemover  }
25692e3bfefSLemover  def addr_low_from_vpn(vpn: UInt) = {
25792e3bfefSLemover    vpn(log2Ceil(l2tlbParams.blockBytes)-log2Ceil(XLEN/8)-1, 0)
25892e3bfefSLemover  }
25992e3bfefSLemover  def addr_low_from_paddr(paddr: UInt) = {
26092e3bfefSLemover    paddr(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8))
26192e3bfefSLemover  }
262d0de7e4aSpeixiaokun  def from_llptw(id: UInt) = {
263d0de7e4aSpeixiaokun    id < l2tlbParams.llptwsize.U
264d0de7e4aSpeixiaokun  }
265d0de7e4aSpeixiaokun  def from_ptw(id: UInt) = {
266d0de7e4aSpeixiaokun    id === l2tlbParams.llptwsize.U
267d0de7e4aSpeixiaokun  }
268d0de7e4aSpeixiaokun  def from_hptw(id: UInt) = {
269d0de7e4aSpeixiaokun    id === l2tlbParams.llptwsize.U + 1.U
27092e3bfefSLemover  }
27192e3bfefSLemover  val waiting_resp = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B)))
27292e3bfefSLemover  val flush_latch = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B)))
27392e3bfefSLemover  for (i <- waiting_resp.indices) {
27492e3bfefSLemover    assert(!flush_latch(i) || waiting_resp(i)) // when sfence_latch wait for mem resp, waiting_resp should be true
27592e3bfefSLemover  }
27692e3bfefSLemover
27792e3bfefSLemover  val llptw_out = llptw.io.out
27892e3bfefSLemover  val llptw_mem = llptw.io.mem
27992e3bfefSLemover  llptw_mem.req_mask := waiting_resp.take(l2tlbParams.llptwsize)
280d0de7e4aSpeixiaokun  ptw.io.mem.mask := waiting_resp.slice(l2tlbParams.llptwsize, l2tlbParams.llptwsize+1)
281d0de7e4aSpeixiaokun  hptw.io.mem.mask := waiting_resp.slice(l2tlbParams.llptwsize+1, l2tlbParams.llptwsize+2)
28292e3bfefSLemover
283d0de7e4aSpeixiaokun  val mem_arb = Module(new Arbiter(new L2TlbMemReqBundle(), 3))
28492e3bfefSLemover  mem_arb.io.in(0) <> ptw.io.mem.req
28592e3bfefSLemover  mem_arb.io.in(1) <> llptw_mem.req
286d0de7e4aSpeixiaokun  mem_arb.io.in(2) <> hptw.io.mem.req
28792e3bfefSLemover  mem_arb.io.out.ready := mem.a.ready && !flush
28892e3bfefSLemover
2891f4a7c0cSLemover  // assert, should not send mem access at same addr for twice.
2907797f035SbugGenerator  val last_resp_vpn = RegEnable(cache.io.refill.bits.req_info_dup(0).vpn, cache.io.refill.valid)
2917797f035SbugGenerator  val last_resp_level = RegEnable(cache.io.refill.bits.level_dup(0), cache.io.refill.valid)
2921f4a7c0cSLemover  val last_resp_v = RegInit(false.B)
293dd7fe201SHaoyuan Feng  val last_has_invalid = !Cat(cache.io.refill.bits.ptes.asTypeOf(Vec(blockBits/XLEN, UInt(XLEN.W))).map(a => a(0))).andR || cache.io.refill.bits.sel_pte_dup(0).asTypeOf(new PteBundle).isAf()
2941f4a7c0cSLemover  when (cache.io.refill.valid) { last_resp_v := !last_has_invalid}
2951f4a7c0cSLemover  when (flush) { last_resp_v := false.B }
2961f4a7c0cSLemover  XSError(last_resp_v && cache.io.refill.valid &&
2977797f035SbugGenerator    (cache.io.refill.bits.req_info_dup(0).vpn === last_resp_vpn) &&
2987797f035SbugGenerator    (cache.io.refill.bits.level_dup(0) === last_resp_level),
2991f4a7c0cSLemover    "l2tlb should not access mem at same addr for twice")
300d0de7e4aSpeixiaokun  // ATTENTION: this may wrongly assert when: a ptes is l2, last part is valid,
3011f4a7c0cSLemover  // but the current part is invalid, so one more mem access happened
3021f4a7c0cSLemover  // If this happened, remove the assert.
3031f4a7c0cSLemover
30492e3bfefSLemover  val req_addr_low = Reg(Vec(MemReqWidth, UInt((log2Up(l2tlbParams.blockBytes)-log2Up(XLEN/8)).W)))
30592e3bfefSLemover
306935edac4STang Haojin  when (llptw.io.in.fire) {
30792e3bfefSLemover    // when enq miss queue, set the req_addr_low to receive the mem resp data part
30892e3bfefSLemover    req_addr_low(llptw_mem.enq_ptr) := addr_low_from_vpn(llptw.io.in.bits.req_info.vpn)
30992e3bfefSLemover  }
310935edac4STang Haojin  when (mem_arb.io.out.fire) {
31192e3bfefSLemover    req_addr_low(mem_arb.io.out.bits.id) := addr_low_from_paddr(mem_arb.io.out.bits.addr)
31292e3bfefSLemover    waiting_resp(mem_arb.io.out.bits.id) := true.B
31392e3bfefSLemover  }
31492e3bfefSLemover  // mem read
31592e3bfefSLemover  val memRead =  edge.Get(
31692e3bfefSLemover    fromSource = mem_arb.io.out.bits.id,
31792e3bfefSLemover    // toAddress  = memAddr(log2Up(CacheLineSize / 2 / 8) - 1, 0),
31892e3bfefSLemover    toAddress  = blockBytes_align(mem_arb.io.out.bits.addr),
31992e3bfefSLemover    lgSize     = log2Up(l2tlbParams.blockBytes).U
32092e3bfefSLemover  )._2
32192e3bfefSLemover  mem.a.bits := memRead
32292e3bfefSLemover  mem.a.valid := mem_arb.io.out.valid && !flush
323d2b20d1aSTang Haojin  mem.a.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.PTW.id.U)
32492e3bfefSLemover  mem.d.ready := true.B
32592e3bfefSLemover  // mem -> data buffer
32692e3bfefSLemover  val refill_data = Reg(Vec(blockBits / l1BusDataWidth, UInt(l1BusDataWidth.W)))
327935edac4STang Haojin  val refill_helper = edge.firstlastHelper(mem.d.bits, mem.d.fire)
32892e3bfefSLemover  val mem_resp_done = refill_helper._3
329d0de7e4aSpeixiaokun  val mem_resp_from_llptw = from_llptw(mem.d.bits.source)
330d0de7e4aSpeixiaokun  val mem_resp_from_ptw = from_ptw(mem.d.bits.source)
331d0de7e4aSpeixiaokun  val mem_resp_from_hptw = from_hptw(mem.d.bits.source)
33292e3bfefSLemover  when (mem.d.valid) {
333d0de7e4aSpeixiaokun    assert(mem.d.bits.source < MemReqWidth.U)
33492e3bfefSLemover    refill_data(refill_helper._4) := mem.d.bits.data
33592e3bfefSLemover  }
3367797f035SbugGenerator  // refill_data_tmp is the wire fork of refill_data, but one cycle earlier
3377797f035SbugGenerator  val refill_data_tmp = WireInit(refill_data)
3387797f035SbugGenerator  refill_data_tmp(refill_helper._4) := mem.d.bits.data
3397797f035SbugGenerator
34092e3bfefSLemover  // save only one pte for each id
34192e3bfefSLemover  // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache)
34292e3bfefSLemover  val resp_pte = VecInit((0 until MemReqWidth).map(i =>
343d0de7e4aSpeixiaokun    if (i == l2tlbParams.llptwsize + 1) {RegEnable(get_part(refill_data_tmp, req_addr_low(i)), mem_resp_done && mem_resp_from_hptw) }
344d0de7e4aSpeixiaokun    else if (i == l2tlbParams.llptwsize) {RegEnable(get_part(refill_data_tmp, req_addr_low(i)), mem_resp_done && mem_resp_from_ptw) }
34592e3bfefSLemover    else { DataHoldBypass(get_part(refill_data, req_addr_low(i)), llptw_mem.buffer_it(i)) }
3467797f035SbugGenerator    // llptw could not use refill_data_tmp, because enq bypass's result works at next cycle
34792e3bfefSLemover  ))
34892e3bfefSLemover
34963632028SHaoyuan Feng  // save eight ptes for each id when sector tlb
35063632028SHaoyuan Feng  // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache)
35163632028SHaoyuan Feng  val resp_pte_sector = VecInit((0 until MemReqWidth).map(i =>
352d0de7e4aSpeixiaokun    if (i == l2tlbParams.llptwsize + 1) {RegEnable(refill_data_tmp, mem_resp_done && mem_resp_from_hptw) }
353d0de7e4aSpeixiaokun    else if (i == l2tlbParams.llptwsize) {RegEnable(refill_data_tmp, mem_resp_done && mem_resp_from_ptw) }
35463632028SHaoyuan Feng    else { DataHoldBypass(refill_data, llptw_mem.buffer_it(i)) }
35563632028SHaoyuan Feng    // llptw could not use refill_data_tmp, because enq bypass's result works at next cycle
35663632028SHaoyuan Feng  ))
35763632028SHaoyuan Feng
358d0de7e4aSpeixiaokun  // mem -> llptw
359d0de7e4aSpeixiaokun  llptw_mem.resp.valid := mem_resp_done && mem_resp_from_llptw
3607797f035SbugGenerator  llptw_mem.resp.bits.id := DataHoldBypass(mem.d.bits.source, mem.d.valid)
36192e3bfefSLemover  // mem -> ptw
36292e3bfefSLemover  ptw.io.mem.req.ready := mem.a.ready
363d0de7e4aSpeixiaokun  ptw.io.mem.resp.valid := mem_resp_done && mem_resp_from_ptw
364d0de7e4aSpeixiaokun  ptw.io.mem.resp.bits := resp_pte.slice(l2tlbParams.llptwsize, l2tlbParams.llptwsize + 1)
365d0de7e4aSpeixiaokun  // mem -> hptw
366d0de7e4aSpeixiaokun  hptw.io.mem.req.ready := mem.a.ready
367d0de7e4aSpeixiaokun  hptw.io.mem.resp.valid := mem_resp_done && mem_resp_from_hptw
368d0de7e4aSpeixiaokun  hptw.io.mem.resp.bits := resp_pte.slice(l2tlbParams.llptwsize + 1, l2tlbParams.llptwsize + 2)
36992e3bfefSLemover  // mem -> cache
370d0de7e4aSpeixiaokun  val refill_from_llptw = mem_resp_from_llptw
371d0de7e4aSpeixiaokun  val refill_from_ptw = mem_resp_from_ptw
372d0de7e4aSpeixiaokun  val refill_from_hptw = mem_resp_from_hptw
373d0de7e4aSpeixiaokun  val refill_level = Mux(refill_from_llptw, 2.U, Mux(refill_from_ptw, RegEnable(ptw.io.refill.level, init = 0.U, ptw.io.mem.req.fire()), RegEnable(hptw.io.refill.level, init = 0.U, hptw.io.mem.req.fire())))
3747797f035SbugGenerator  val refill_valid = mem_resp_done && !flush && !flush_latch(mem.d.bits.source)
3757797f035SbugGenerator
3767797f035SbugGenerator  cache.io.refill.valid := RegNext(refill_valid, false.B)
37792e3bfefSLemover  cache.io.refill.bits.ptes := refill_data.asUInt
378d0de7e4aSpeixiaokun  cache.io.refill.bits.req_info_dup.map(_ := RegEnable(Mux(refill_from_llptw, llptw_mem.refill, Mux(refill_from_ptw, ptw.io.refill.req_info, hptw.io.refill.req_info)), refill_valid))
3797797f035SbugGenerator  cache.io.refill.bits.level_dup.map(_ := RegEnable(refill_level, refill_valid))
3807797f035SbugGenerator  cache.io.refill.bits.levelOH(refill_level, refill_valid)
3817797f035SbugGenerator  cache.io.refill.bits.sel_pte_dup.map(_ := RegNext(sel_data(refill_data_tmp.asUInt, req_addr_low(mem.d.bits.source))))
38292e3bfefSLemover
3839c26bab7SHaoyuan Feng  if (env.EnableDifftest) {
3849c26bab7SHaoyuan Feng    val difftest_ptw_addr = RegInit(VecInit(Seq.fill(MemReqWidth)(0.U(PAddrBits.W))))
3859c26bab7SHaoyuan Feng    when (mem.a.valid) {
3869c26bab7SHaoyuan Feng      difftest_ptw_addr(mem.a.bits.source) := mem.a.bits.address
3879c26bab7SHaoyuan Feng    }
3889c26bab7SHaoyuan Feng
389a0c65233SYinan Xu    val difftest = DifftestModule(new DiffRefillEvent, dontCare = true)
390254e4960SHaoyuan Feng    difftest.coreid := io.hartId
3917d45a146SYinan Xu    difftest.index := 2.U
3927d45a146SYinan Xu    difftest.valid := cache.io.refill.valid
3937d45a146SYinan Xu    difftest.addr := difftest_ptw_addr(RegNext(mem.d.bits.source))
3947d45a146SYinan Xu    difftest.data := refill_data.asTypeOf(difftest.data)
395935edac4STang Haojin    difftest.idtfr := DontCare
3969c26bab7SHaoyuan Feng  }
3979c26bab7SHaoyuan Feng
3985ab1b84dSHaoyuan Feng  if (env.EnableDifftest) {
3995ab1b84dSHaoyuan Feng    for (i <- 0 until PtwWidth) {
4007d45a146SYinan Xu      val difftest = DifftestModule(new DiffL2TLBEvent)
401254e4960SHaoyuan Feng      difftest.coreid := io.hartId
4027d45a146SYinan Xu      difftest.valid := io.tlb(i).resp.fire && !io.tlb(i).resp.bits.af
4037d45a146SYinan Xu      difftest.index := i.U
4047d45a146SYinan Xu      difftest.satp := io.csr.tlb.satp.ppn
4057d45a146SYinan Xu      difftest.vpn := Cat(io.tlb(i).resp.bits.entry.tag, 0.U(sectortlbwidth.W))
40663632028SHaoyuan Feng      for (j <- 0 until tlbcontiguous) {
4077d45a146SYinan Xu        difftest.ppn(j) := Cat(io.tlb(i).resp.bits.entry.ppn, io.tlb(i).resp.bits.ppn_low(j))
4087d45a146SYinan Xu        difftest.valididx(j) := io.tlb(i).resp.bits.valididx(j)
409*82978df9Speixiaokun        difftest.io.pteidx(j) := io.tlb(i).resp.bits.s1.pteidx(j)
41063632028SHaoyuan Feng      }
4117d45a146SYinan Xu      difftest.perm := io.tlb(i).resp.bits.entry.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt
4127d45a146SYinan Xu      difftest.level := io.tlb(i).resp.bits.entry.level.getOrElse(0.U.asUInt)
4137d45a146SYinan Xu      difftest.pf := io.tlb(i).resp.bits.pf
4145ab1b84dSHaoyuan Feng    }
4155ab1b84dSHaoyuan Feng  }
4165ab1b84dSHaoyuan Feng
41792e3bfefSLemover  // pmp
41892e3bfefSLemover  pmp_check(0).req <> ptw.io.pmp.req
41992e3bfefSLemover  ptw.io.pmp.resp <> pmp_check(0).resp
42092e3bfefSLemover  pmp_check(1).req <> llptw.io.pmp.req
42192e3bfefSLemover  llptw.io.pmp.resp <> pmp_check(1).resp
42292e3bfefSLemover
42392e3bfefSLemover  llptw_out.ready := outReady(llptw_out.bits.req_info.source, outArbMqPort)
42463632028SHaoyuan Feng
425d0de7e4aSpeixiaokun  // hptw and page cache -> ptw and llptw
426d0de7e4aSpeixiaokun  val HptwRespArbCachePort = 0
427d0de7e4aSpeixiaokun  val HptwRespArbHptw = 0
428d0de7e4aSpeixiaokun  hptw_resp_arb.io.in(HptwRespArbCachePort).valid := cache.io.resp.valid && cache.io.resp.bits.hit && cache.io.resp.bits.isHptw
429d0de7e4aSpeixiaokun  hptw_resp_arb.io.in(HptwRespArbCachePort).bits.id := cache.io.resp.bits.toHptw.id
430d0de7e4aSpeixiaokun  hptw_resp_arb.io.in(HptwRespArbCachePort).bits.resp := cache.io.resp.bits.toHptw.resp
431d0de7e4aSpeixiaokun  hptw_resp_arb.io.in(HptwRespArbHptw).valid := hptw.io.resp.valid
432d0de7e4aSpeixiaokun  hptw_resp_arb.io.in(HptwRespArbHptw).bits.id := hptw.io.resp.bits.id
433d0de7e4aSpeixiaokun  hptw_resp_arb.io.in(HptwRespArbHptw).bits.resp := hptw.io.resp.bits.resp
434d0de7e4aSpeixiaokun
435d0de7e4aSpeixiaokun  ptw.io.hptw.resp.valid := hptw_resp_arb.io.out.valid && hptw_resp_arb.io.out.bits.id === FsmReqID.U
436d0de7e4aSpeixiaokun  ptw.io.hptw.resp.bits.h_resp := hptw_resp_arb.io.out.bits.resp
437d0de7e4aSpeixiaokun  llptw.io.hptw.resp.valid := hptw_resp_arb.io.out.valid && hptw_resp_arb.io.out.bits.id =/= FsmReqID.U
438d0de7e4aSpeixiaokun  llptw.io.hptw.resp.bits.h_resp := hptw_resp_arb.io.out.bits.resp
439d0de7e4aSpeixiaokun
44063632028SHaoyuan Feng  // Timing: Maybe need to do some optimization or even add one more cycle
44192e3bfefSLemover  for (i <- 0 until PtwWidth) {
44263632028SHaoyuan Feng    mergeArb(i).in(outArbCachePort).valid := cache.io.resp.valid && cache.io.resp.bits.hit && cache.io.resp.bits.req_info.source===i.U
443d0de7e4aSpeixiaokun    mergeArb(i).in(outArbCachePort).bits.s2xlate := cache.io.resp.bits.req_info.s2xlate
444d0de7e4aSpeixiaokun    mergeArb(i).in(outArbCachePort).bits.s1Resp := cache.io.resp.bits.toTlb
445d0de7e4aSpeixiaokun    mergeArb(i).in(outArbCachePort).bits.s2Resp := cache.io.resp.bits.toHptw.resp
44663632028SHaoyuan Feng    mergeArb(i).in(outArbFsmPort).valid := ptw.io.resp.valid && ptw.io.resp.bits.source===i.U
447d0de7e4aSpeixiaokun    mergeArb(i).in(outArbFsmPort).bits.s2xlate := ptw.io.resp.bits.s2xlate
448d0de7e4aSpeixiaokun    mergeArb(i).in(outArbFsmPort).bits.s1Resp := ptw.io.resp.bits.resp
449d0de7e4aSpeixiaokun    mergeArb(i).in(outArbFsmPort).bits.s2Resp := ptw.io.resp.bits.h_resp
45063632028SHaoyuan Feng    mergeArb(i).in(outArbMqPort).valid := llptw_out.valid && llptw_out.bits.req_info.source===i.U
451d0de7e4aSpeixiaokun    mergeArb(i).in(outArbMqPort).bits.s2xlate := llptw_out.bits.req_info.s2xlate
452d0de7e4aSpeixiaokun    mergeArb(i).in(outArbMqPort).bits.s1Resp := contiguous_pte_to_merge_ptwResp(resp_pte_sector(llptw_out.bits.id).asUInt, llptw_out.bits.req_info.vpn, llptw_out.bits.af, true)
453d0de7e4aSpeixiaokun    mergeArb(i).in(outArbMqPort).bits.s2Resp := llptw_out.bits.h_resp
45463632028SHaoyuan Feng    mergeArb(i).out.ready := outArb(i).in(0).ready
45563632028SHaoyuan Feng  }
45663632028SHaoyuan Feng
45763632028SHaoyuan Feng  for (i <- 0 until PtwWidth) {
45863632028SHaoyuan Feng    outArb(i).in(0).valid := mergeArb(i).out.valid
459d0de7e4aSpeixiaokun    outArb(i).in(0).bits.s1Resp := merge_ptwResp_to_sector_ptwResp(mergeArb(i).out.bits.s1Resp)
460d0de7e4aSpeixiaokun    outArb(i).in(0).bits.s2Resp := mergeArb(i).out.bits.s2Resp
46192e3bfefSLemover  }
46292e3bfefSLemover
46392e3bfefSLemover  // io.tlb.map(_.resp) <> outArb.map(_.out)
46492e3bfefSLemover  io.tlb.map(_.resp).zip(outArb.map(_.out)).map{
46592e3bfefSLemover    case (resp, out) => resp <> out
46692e3bfefSLemover  }
46792e3bfefSLemover
46892e3bfefSLemover  // sfence
46992e3bfefSLemover  when (flush) {
47092e3bfefSLemover    for (i <- 0 until MemReqWidth) {
47192e3bfefSLemover      when (waiting_resp(i)) {
47292e3bfefSLemover        flush_latch(i) := true.B
47392e3bfefSLemover      }
47492e3bfefSLemover    }
47592e3bfefSLemover  }
47692e3bfefSLemover  // mem -> control signal
47792e3bfefSLemover  // waiting_resp and sfence_latch will be reset when mem_resp_done
47892e3bfefSLemover  when (mem_resp_done) {
47992e3bfefSLemover    waiting_resp(mem.d.bits.source) := false.B
48092e3bfefSLemover    flush_latch(mem.d.bits.source) := false.B
48192e3bfefSLemover  }
48292e3bfefSLemover
48392e3bfefSLemover  def block_decoupled[T <: Data](source: DecoupledIO[T], sink: DecoupledIO[T], block_signal: Bool) = {
48492e3bfefSLemover    sink.valid   := source.valid && !block_signal
48592e3bfefSLemover    source.ready := sink.ready   && !block_signal
48692e3bfefSLemover    sink.bits    := source.bits
48792e3bfefSLemover  }
48892e3bfefSLemover
48992e3bfefSLemover  def get_part(data: Vec[UInt], index: UInt): UInt = {
49092e3bfefSLemover    val inner_data = data.asTypeOf(Vec(data.getWidth / XLEN, UInt(XLEN.W)))
49192e3bfefSLemover    inner_data(index)
49292e3bfefSLemover  }
49392e3bfefSLemover
49463632028SHaoyuan Feng  // not_super means that this is a normal page
49563632028SHaoyuan Feng  // valididx(i) will be all true when super page to be convenient for l1 tlb matching
49663632028SHaoyuan Feng  def contiguous_pte_to_merge_ptwResp(pte: UInt, vpn: UInt, af: Bool, af_first: Boolean, not_super: Boolean = true) : PtwMergeResp = {
49763632028SHaoyuan Feng    assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!")
49863632028SHaoyuan Feng    val ptw_merge_resp = Wire(new PtwMergeResp())
49963632028SHaoyuan Feng    for (i <- 0 until tlbcontiguous) {
50063632028SHaoyuan Feng      val pte_in = pte(64 * i + 63, 64 * i).asTypeOf(new PteBundle())
50163632028SHaoyuan Feng      val ptw_resp = Wire(new PtwMergeEntry(tagLen = sectorvpnLen, hasPerm = true, hasLevel = true))
50263632028SHaoyuan Feng      ptw_resp.ppn := pte_in.ppn(ppnLen - 1, sectortlbwidth)
50363632028SHaoyuan Feng      ptw_resp.ppn_low := pte_in.ppn(sectortlbwidth - 1, 0)
50463632028SHaoyuan Feng      ptw_resp.level.map(_ := 2.U)
50563632028SHaoyuan Feng      ptw_resp.perm.map(_ := pte_in.getPerm())
50663632028SHaoyuan Feng      ptw_resp.tag := vpn(vpnLen - 1, sectortlbwidth)
50763632028SHaoyuan Feng      ptw_resp.pf := (if (af_first) !af else true.B) && pte_in.isPf(2.U)
50863632028SHaoyuan Feng      ptw_resp.af := (if (!af_first) pte_in.isPf(2.U) else true.B) && (af || pte_in.isAf())
50963632028SHaoyuan Feng      ptw_resp.v := !ptw_resp.pf
51063632028SHaoyuan Feng      ptw_resp.prefetch := DontCare
51163632028SHaoyuan Feng      ptw_resp.asid := satp.asid
51263632028SHaoyuan Feng      ptw_merge_resp.entry(i) := ptw_resp
51363632028SHaoyuan Feng    }
51463632028SHaoyuan Feng    ptw_merge_resp.pteidx := UIntToOH(vpn(sectortlbwidth - 1, 0)).asBools
51563632028SHaoyuan Feng    ptw_merge_resp.not_super := not_super.B
51663632028SHaoyuan Feng    ptw_merge_resp
51763632028SHaoyuan Feng  }
51863632028SHaoyuan Feng
51963632028SHaoyuan Feng  def merge_ptwResp_to_sector_ptwResp(pte: PtwMergeResp) : PtwSectorResp = {
52063632028SHaoyuan Feng    assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!")
52163632028SHaoyuan Feng    val ptw_sector_resp = Wire(new PtwSectorResp)
52263632028SHaoyuan Feng    ptw_sector_resp.entry.tag := pte.entry(OHToUInt(pte.pteidx)).tag
52363632028SHaoyuan Feng    ptw_sector_resp.entry.asid := pte.entry(OHToUInt(pte.pteidx)).asid
52463632028SHaoyuan Feng    ptw_sector_resp.entry.ppn := pte.entry(OHToUInt(pte.pteidx)).ppn
52563632028SHaoyuan Feng    ptw_sector_resp.entry.perm.map(_ := pte.entry(OHToUInt(pte.pteidx)).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)))
52663632028SHaoyuan Feng    ptw_sector_resp.entry.level.map(_ := pte.entry(OHToUInt(pte.pteidx)).level.getOrElse(0.U(2.W)))
52763632028SHaoyuan Feng    ptw_sector_resp.entry.prefetch := pte.entry(OHToUInt(pte.pteidx)).prefetch
52863632028SHaoyuan Feng    ptw_sector_resp.entry.v := pte.entry(OHToUInt(pte.pteidx)).v
52963632028SHaoyuan Feng    ptw_sector_resp.af := pte.entry(OHToUInt(pte.pteidx)).af
53063632028SHaoyuan Feng    ptw_sector_resp.pf := pte.entry(OHToUInt(pte.pteidx)).pf
53163632028SHaoyuan Feng    ptw_sector_resp.addr_low := OHToUInt(pte.pteidx)
532b0fa7106SHaoyuan Feng    ptw_sector_resp.pteidx := pte.pteidx
53363632028SHaoyuan Feng    for (i <- 0 until tlbcontiguous) {
53463632028SHaoyuan Feng      val ppn_equal = pte.entry(i).ppn === pte.entry(OHToUInt(pte.pteidx)).ppn
53563632028SHaoyuan Feng      val perm_equal = pte.entry(i).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt === pte.entry(OHToUInt(pte.pteidx)).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt
53663632028SHaoyuan Feng      val v_equal = pte.entry(i).v === pte.entry(OHToUInt(pte.pteidx)).v
53763632028SHaoyuan Feng      val af_equal = pte.entry(i).af === pte.entry(OHToUInt(pte.pteidx)).af
53863632028SHaoyuan Feng      val pf_equal = pte.entry(i).pf === pte.entry(OHToUInt(pte.pteidx)).pf
53963632028SHaoyuan Feng      ptw_sector_resp.valididx(i) := (ppn_equal && perm_equal && v_equal && af_equal && pf_equal) || !pte.not_super
54063632028SHaoyuan Feng      ptw_sector_resp.ppn_low(i) := pte.entry(i).ppn_low
54163632028SHaoyuan Feng    }
54263632028SHaoyuan Feng    ptw_sector_resp.valididx(OHToUInt(pte.pteidx)) := true.B
54363632028SHaoyuan Feng    ptw_sector_resp
54463632028SHaoyuan Feng  }
54563632028SHaoyuan Feng
54692e3bfefSLemover  def outReady(source: UInt, port: Int): Bool = {
54745f43e6eSTang Haojin    MuxLookup(source, true.B)((0 until PtwWidth).map(i => i.U -> mergeArb(i).in(port).ready))
54892e3bfefSLemover  }
54992e3bfefSLemover
55092e3bfefSLemover  // debug info
55192e3bfefSLemover  for (i <- 0 until PtwWidth) {
55292e3bfefSLemover    XSDebug(p"[io.tlb(${i.U})] ${io.tlb(i)}\n")
55392e3bfefSLemover  }
5547797f035SbugGenerator  XSDebug(p"[sfence] ${io.sfence}\n")
55592e3bfefSLemover  XSDebug(p"[io.csr.tlb] ${io.csr.tlb}\n")
55692e3bfefSLemover
55792e3bfefSLemover  for (i <- 0 until PtwWidth) {
558935edac4STang Haojin    XSPerfAccumulate(s"req_count${i}", io.tlb(i).req(0).fire)
55992e3bfefSLemover    XSPerfAccumulate(s"req_blocked_count_${i}", io.tlb(i).req(0).valid && !io.tlb(i).req(0).ready)
56092e3bfefSLemover  }
56192e3bfefSLemover  XSPerfAccumulate(s"req_blocked_by_mq", arb1.io.out.valid && missQueue.io.out.valid)
56292e3bfefSLemover  for (i <- 0 until (MemReqWidth + 1)) {
56392e3bfefSLemover    XSPerfAccumulate(s"mem_req_util${i}", PopCount(waiting_resp) === i.U)
56492e3bfefSLemover  }
56592e3bfefSLemover  XSPerfAccumulate("mem_cycle", PopCount(waiting_resp) =/= 0.U)
566935edac4STang Haojin  XSPerfAccumulate("mem_count", mem.a.fire)
567dd7fe201SHaoyuan Feng  for (i <- 0 until PtwWidth) {
56863632028SHaoyuan Feng    XSPerfAccumulate(s"llptw_ppn_af${i}", mergeArb(i).in(outArbMqPort).valid && mergeArb(i).in(outArbMqPort).bits.entry(OHToUInt(mergeArb(i).in(outArbMqPort).bits.pteidx)).af && !llptw_out.bits.af)
569dd7fe201SHaoyuan Feng    XSPerfAccumulate(s"access_fault${i}", io.tlb(i).resp.fire && io.tlb(i).resp.bits.af)
570dd7fe201SHaoyuan Feng  }
57192e3bfefSLemover
57292e3bfefSLemover  // print configs
573f1fe8698SLemover  println(s"${l2tlbParams.name}: a ptw, a llptw with size ${l2tlbParams.llptwsize}, miss queue size ${MissQueueSize} l1:${l2tlbParams.l1Size} fa l2: nSets ${l2tlbParams.l2nSets} nWays ${l2tlbParams.l2nWays} l3: ${l2tlbParams.l3nSets} nWays ${l2tlbParams.l3nWays} blockBytes:${l2tlbParams.blockBytes}")
57492e3bfefSLemover
57592e3bfefSLemover  // time out assert
57692e3bfefSLemover  for (i <- 0 until MemReqWidth) {
57792e3bfefSLemover    TimeOutAssert(waiting_resp(i), timeOutThreshold, s"ptw mem resp time out wait_resp${i}")
57892e3bfefSLemover    TimeOutAssert(flush_latch(i), timeOutThreshold, s"ptw mem resp time out flush_latch${i}")
57992e3bfefSLemover  }
58092e3bfefSLemover
58192e3bfefSLemover
58292e3bfefSLemover  val perfEvents  = Seq(llptw, cache, ptw).flatMap(_.getPerfEvents)
58392e3bfefSLemover  generatePerfEvent()
5845afdf73cSHaoyuan Feng
585da3bf434SMaxpicca-Li  val isWriteL1TlbTable = WireInit(Constantin.createRecord("isWriteL1TlbTable" + p(XSCoreParamsKey).HartId.toString))
5865afdf73cSHaoyuan Feng  val L1TlbTable = ChiselDB.createTable("L1Tlb_hart" + p(XSCoreParamsKey).HartId.toString, new L1TlbDB)
5875afdf73cSHaoyuan Feng  val ITlbReqDB, DTlbReqDB, ITlbRespDB, DTlbRespDB = Wire(new L1TlbDB)
5885afdf73cSHaoyuan Feng  ITlbReqDB.vpn := io.tlb(0).req(0).bits.vpn
5895afdf73cSHaoyuan Feng  DTlbReqDB.vpn := io.tlb(1).req(0).bits.vpn
5905afdf73cSHaoyuan Feng  ITlbRespDB.vpn := io.tlb(0).resp.bits.entry.tag
5915afdf73cSHaoyuan Feng  DTlbRespDB.vpn := io.tlb(1).resp.bits.entry.tag
592da3bf434SMaxpicca-Li  L1TlbTable.log(ITlbReqDB, isWriteL1TlbTable.orR && io.tlb(0).req(0).fire, "ITlbReq", clock, reset)
593da3bf434SMaxpicca-Li  L1TlbTable.log(DTlbReqDB, isWriteL1TlbTable.orR && io.tlb(1).req(0).fire, "DTlbReq", clock, reset)
594da3bf434SMaxpicca-Li  L1TlbTable.log(ITlbRespDB, isWriteL1TlbTable.orR && io.tlb(0).resp.fire, "ITlbResp", clock, reset)
595da3bf434SMaxpicca-Li  L1TlbTable.log(DTlbRespDB, isWriteL1TlbTable.orR && io.tlb(1).resp.fire, "DTlbResp", clock, reset)
5965afdf73cSHaoyuan Feng
597da3bf434SMaxpicca-Li  val isWritePageCacheTable = WireInit(Constantin.createRecord("isWritePageCacheTable" + p(XSCoreParamsKey).HartId.toString))
5985afdf73cSHaoyuan Feng  val PageCacheTable = ChiselDB.createTable("PageCache_hart" + p(XSCoreParamsKey).HartId.toString, new PageCacheDB)
5995afdf73cSHaoyuan Feng  val PageCacheDB = Wire(new PageCacheDB)
60063632028SHaoyuan Feng  PageCacheDB.vpn := Cat(cache.io.resp.bits.toTlb.entry(0).tag, OHToUInt(cache.io.resp.bits.toTlb.pteidx))
6015afdf73cSHaoyuan Feng  PageCacheDB.source := cache.io.resp.bits.req_info.source
6025afdf73cSHaoyuan Feng  PageCacheDB.bypassed := cache.io.resp.bits.bypassed
6035afdf73cSHaoyuan Feng  PageCacheDB.is_first := cache.io.resp.bits.isFirst
60463632028SHaoyuan Feng  PageCacheDB.prefetched := cache.io.resp.bits.toTlb.entry(0).prefetch
6055afdf73cSHaoyuan Feng  PageCacheDB.prefetch := cache.io.resp.bits.prefetch
6065afdf73cSHaoyuan Feng  PageCacheDB.l2Hit := cache.io.resp.bits.toFsm.l2Hit
6075afdf73cSHaoyuan Feng  PageCacheDB.l1Hit := cache.io.resp.bits.toFsm.l1Hit
6085afdf73cSHaoyuan Feng  PageCacheDB.hit := cache.io.resp.bits.hit
609da3bf434SMaxpicca-Li  PageCacheTable.log(PageCacheDB, isWritePageCacheTable.orR && cache.io.resp.fire, "PageCache", clock, reset)
6105afdf73cSHaoyuan Feng
611da3bf434SMaxpicca-Li  val isWritePTWTable = WireInit(Constantin.createRecord("isWritePTWTable" + p(XSCoreParamsKey).HartId.toString))
6125afdf73cSHaoyuan Feng  val PTWTable = ChiselDB.createTable("PTW_hart" + p(XSCoreParamsKey).HartId.toString, new PTWDB)
6135afdf73cSHaoyuan Feng  val PTWReqDB, PTWRespDB, LLPTWReqDB, LLPTWRespDB = Wire(new PTWDB)
6145afdf73cSHaoyuan Feng  PTWReqDB.vpn := ptw.io.req.bits.req_info.vpn
6155afdf73cSHaoyuan Feng  PTWReqDB.source := ptw.io.req.bits.req_info.source
6165afdf73cSHaoyuan Feng  PTWRespDB.vpn := ptw.io.refill.req_info.vpn
6175afdf73cSHaoyuan Feng  PTWRespDB.source := ptw.io.refill.req_info.source
6185afdf73cSHaoyuan Feng  LLPTWReqDB.vpn := llptw.io.in.bits.req_info.vpn
6195afdf73cSHaoyuan Feng  LLPTWReqDB.source := llptw.io.in.bits.req_info.source
6205afdf73cSHaoyuan Feng  LLPTWRespDB.vpn := llptw.io.mem.refill.vpn
6215afdf73cSHaoyuan Feng  LLPTWRespDB.source := llptw.io.mem.refill.source
622da3bf434SMaxpicca-Li  PTWTable.log(PTWReqDB, isWritePTWTable.orR && ptw.io.req.fire, "PTWReq", clock, reset)
623da3bf434SMaxpicca-Li  PTWTable.log(PTWRespDB, isWritePTWTable.orR && ptw.io.mem.resp.fire, "PTWResp", clock, reset)
624da3bf434SMaxpicca-Li  PTWTable.log(LLPTWReqDB, isWritePTWTable.orR && llptw.io.in.fire, "LLPTWReq", clock, reset)
625da3bf434SMaxpicca-Li  PTWTable.log(LLPTWRespDB, isWritePTWTable.orR && llptw.io.mem.resp.fire, "LLPTWResp", clock, reset)
6265afdf73cSHaoyuan Feng
627da3bf434SMaxpicca-Li  val isWriteL2TlbMissQueueTable = WireInit(Constantin.createRecord("isWriteL2TlbMissQueueTable" + p(XSCoreParamsKey).HartId.toString))
6285afdf73cSHaoyuan Feng  val L2TlbMissQueueTable = ChiselDB.createTable("L2TlbMissQueue_hart" + p(XSCoreParamsKey).HartId.toString, new L2TlbMissQueueDB)
6295afdf73cSHaoyuan Feng  val L2TlbMissQueueInDB, L2TlbMissQueueOutDB = Wire(new L2TlbMissQueueDB)
6305afdf73cSHaoyuan Feng  L2TlbMissQueueInDB.vpn := missQueue.io.in.bits.vpn
6315afdf73cSHaoyuan Feng  L2TlbMissQueueOutDB.vpn := missQueue.io.out.bits.vpn
632da3bf434SMaxpicca-Li  L2TlbMissQueueTable.log(L2TlbMissQueueInDB, isWriteL2TlbMissQueueTable.orR && missQueue.io.in.fire, "L2TlbMissQueueIn", clock, reset)
633da3bf434SMaxpicca-Li  L2TlbMissQueueTable.log(L2TlbMissQueueOutDB, isWriteL2TlbMissQueueTable.orR && missQueue.io.out.fire, "L2TlbMissQueueOut", clock, reset)
63492e3bfefSLemover}
63592e3bfefSLemover
6367797f035SbugGenerator/** BlockHelper, block missqueue, not to send too many req to cache
6377797f035SbugGenerator *  Parameter:
6387797f035SbugGenerator *    enable: enable BlockHelper, mq should not send too many reqs
6397797f035SbugGenerator *    start: when miss queue out fire and need, block miss queue's out
6407797f035SbugGenerator *    block: block miss queue's out
6417797f035SbugGenerator *    latency: last missqueue out's cache access latency
6427797f035SbugGenerator */
6437797f035SbugGeneratorclass BlockHelper(latency: Int)(implicit p: Parameters) extends XSModule {
6447797f035SbugGenerator  val io = IO(new Bundle {
6457797f035SbugGenerator    val enable = Input(Bool())
6467797f035SbugGenerator    val start = Input(Bool())
6477797f035SbugGenerator    val block = Output(Bool())
6487797f035SbugGenerator  })
6497797f035SbugGenerator
6507797f035SbugGenerator  val count = RegInit(0.U(log2Ceil(latency).W))
6517797f035SbugGenerator  val valid = RegInit(false.B)
6527797f035SbugGenerator  val work = RegInit(true.B)
6537797f035SbugGenerator
6547797f035SbugGenerator  io.block := valid
6557797f035SbugGenerator
6567797f035SbugGenerator  when (io.start && work) { valid := true.B }
6577797f035SbugGenerator  when (valid) { count := count + 1.U }
6587797f035SbugGenerator  when (count === (latency.U) || io.enable) {
6597797f035SbugGenerator    valid := false.B
6607797f035SbugGenerator    work := io.enable
6617797f035SbugGenerator    count := 0.U
6627797f035SbugGenerator  }
6637797f035SbugGenerator}
6647797f035SbugGenerator
66592e3bfefSLemoverclass PTEHelper() extends ExtModule {
66692e3bfefSLemover  val clock  = IO(Input(Clock()))
66792e3bfefSLemover  val enable = IO(Input(Bool()))
66892e3bfefSLemover  val satp   = IO(Input(UInt(64.W)))
66992e3bfefSLemover  val vpn    = IO(Input(UInt(64.W)))
67092e3bfefSLemover  val pte    = IO(Output(UInt(64.W)))
67192e3bfefSLemover  val level  = IO(Output(UInt(8.W)))
67292e3bfefSLemover  val pf     = IO(Output(UInt(8.W)))
67392e3bfefSLemover}
67492e3bfefSLemover
6755afdf73cSHaoyuan Fengclass PTWDelayN[T <: Data](gen: T, n: Int, flush: Bool) extends Module {
6765afdf73cSHaoyuan Feng  val io = IO(new Bundle() {
6775afdf73cSHaoyuan Feng    val in = Input(gen)
6785afdf73cSHaoyuan Feng    val out = Output(gen)
6795afdf73cSHaoyuan Feng    val ptwflush = Input(flush.cloneType)
6805afdf73cSHaoyuan Feng  })
6815afdf73cSHaoyuan Feng  val out = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(gen))))
6825afdf73cSHaoyuan Feng  val t = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(gen))))
6835afdf73cSHaoyuan Feng  out(0) := io.in
6845afdf73cSHaoyuan Feng  if (n == 1) {
6855afdf73cSHaoyuan Feng    io.out := out(0)
6865afdf73cSHaoyuan Feng  } else {
6875afdf73cSHaoyuan Feng    when (io.ptwflush) {
6885afdf73cSHaoyuan Feng      for (i <- 0 until n) {
6895afdf73cSHaoyuan Feng        t(i) := 0.U.asTypeOf(gen)
6905afdf73cSHaoyuan Feng        out(i) := 0.U.asTypeOf(gen)
6915afdf73cSHaoyuan Feng      }
6925afdf73cSHaoyuan Feng      io.out := 0.U.asTypeOf(gen)
6935afdf73cSHaoyuan Feng    } .otherwise {
6945afdf73cSHaoyuan Feng      for (i <- 1 until n) {
6955afdf73cSHaoyuan Feng        t(i-1) := out(i-1)
6965afdf73cSHaoyuan Feng        out(i) := t(i-1)
6975afdf73cSHaoyuan Feng      }
6985afdf73cSHaoyuan Feng      io.out := out(n-1)
6995afdf73cSHaoyuan Feng    }
7005afdf73cSHaoyuan Feng  }
7015afdf73cSHaoyuan Feng}
7025afdf73cSHaoyuan Feng
7035afdf73cSHaoyuan Fengobject PTWDelayN {
7045afdf73cSHaoyuan Feng  def apply[T <: Data](in: T, n: Int, flush: Bool): T = {
7055afdf73cSHaoyuan Feng    val delay = Module(new PTWDelayN(in.cloneType, n, flush))
7065afdf73cSHaoyuan Feng    delay.io.in := in
7075afdf73cSHaoyuan Feng    delay.io.ptwflush := flush
7085afdf73cSHaoyuan Feng    delay.io.out
7095afdf73cSHaoyuan Feng  }
7105afdf73cSHaoyuan Feng}
7115afdf73cSHaoyuan Feng
71292e3bfefSLemoverclass FakePTW()(implicit p: Parameters) extends XSModule with HasPtwConst {
71392e3bfefSLemover  val io = IO(new L2TLBIO)
7145afdf73cSHaoyuan Feng  val flush = VecInit(Seq.fill(PtwWidth)(false.B))
7155afdf73cSHaoyuan Feng  flush(0) := DelayN(io.sfence.valid || io.csr.tlb.satp.changed, itlbParams.fenceDelay)
7165afdf73cSHaoyuan Feng  flush(1) := DelayN(io.sfence.valid || io.csr.tlb.satp.changed, ldtlbParams.fenceDelay)
71792e3bfefSLemover  for (i <- 0 until PtwWidth) {
71892e3bfefSLemover    val helper = Module(new PTEHelper())
71992e3bfefSLemover    helper.clock := clock
72092e3bfefSLemover    helper.satp := io.csr.tlb.satp.ppn
7215afdf73cSHaoyuan Feng
7225afdf73cSHaoyuan Feng    if (coreParams.softPTWDelay == 1) {
7235afdf73cSHaoyuan Feng      helper.enable := io.tlb(i).req(0).fire
72492e3bfefSLemover      helper.vpn := io.tlb(i).req(0).bits.vpn
7255afdf73cSHaoyuan Feng    } else {
7265afdf73cSHaoyuan Feng      helper.enable := PTWDelayN(io.tlb(i).req(0).fire, coreParams.softPTWDelay - 1, flush(i))
7275afdf73cSHaoyuan Feng      helper.vpn := PTWDelayN(io.tlb(i).req(0).bits.vpn, coreParams.softPTWDelay - 1, flush(i))
7285afdf73cSHaoyuan Feng    }
7295afdf73cSHaoyuan Feng
73092e3bfefSLemover    val pte = helper.pte.asTypeOf(new PteBundle)
73192e3bfefSLemover    val level = helper.level
73292e3bfefSLemover    val pf = helper.pf
7335afdf73cSHaoyuan Feng    val empty = RegInit(true.B)
7345afdf73cSHaoyuan Feng    when (io.tlb(i).req(0).fire) {
7355afdf73cSHaoyuan Feng      empty := false.B
7365afdf73cSHaoyuan Feng    } .elsewhen (io.tlb(i).resp.fire || flush(i)) {
7375afdf73cSHaoyuan Feng      empty := true.B
7385afdf73cSHaoyuan Feng    }
73992e3bfefSLemover
7405afdf73cSHaoyuan Feng    io.tlb(i).req(0).ready := empty || io.tlb(i).resp.fire
7415afdf73cSHaoyuan Feng    io.tlb(i).resp.valid := PTWDelayN(io.tlb(i).req(0).fire, coreParams.softPTWDelay, flush(i))
74292e3bfefSLemover    assert(!io.tlb(i).resp.valid || io.tlb(i).resp.ready)
7435afdf73cSHaoyuan Feng    io.tlb(i).resp.bits.entry.tag := PTWDelayN(io.tlb(i).req(0).bits.vpn, coreParams.softPTWDelay, flush(i))
74492e3bfefSLemover    io.tlb(i).resp.bits.entry.ppn := pte.ppn
74592e3bfefSLemover    io.tlb(i).resp.bits.entry.perm.map(_ := pte.getPerm())
74692e3bfefSLemover    io.tlb(i).resp.bits.entry.level.map(_ := level)
74792e3bfefSLemover    io.tlb(i).resp.bits.pf := pf
74892e3bfefSLemover    io.tlb(i).resp.bits.af := DontCare // TODO: implement it
7495afdf73cSHaoyuan Feng    io.tlb(i).resp.bits.entry.v := !pf
7505afdf73cSHaoyuan Feng    io.tlb(i).resp.bits.entry.prefetch := DontCare
7515afdf73cSHaoyuan Feng    io.tlb(i).resp.bits.entry.asid := io.csr.tlb.satp.asid
75292e3bfefSLemover  }
75392e3bfefSLemover}
75492e3bfefSLemover
75592e3bfefSLemoverclass L2TLBWrapper()(implicit p: Parameters) extends LazyModule with HasXSParameter {
75695e60e55STang Haojin  override def shouldBeInlined: Boolean = false
75792e3bfefSLemover  val useSoftPTW = coreParams.softPTW
75892e3bfefSLemover  val node = if (!useSoftPTW) TLIdentityNode() else null
75992e3bfefSLemover  val ptw = if (!useSoftPTW) LazyModule(new L2TLB()) else null
76092e3bfefSLemover  if (!useSoftPTW) {
76192e3bfefSLemover    node := ptw.node
76292e3bfefSLemover  }
76392e3bfefSLemover
764935edac4STang Haojin  class L2TLBWrapperImp(wrapper: LazyModule) extends LazyModuleImp(wrapper) with HasPerfEvents {
76592e3bfefSLemover    val io = IO(new L2TLBIO)
76692e3bfefSLemover    val perfEvents = if (useSoftPTW) {
76792e3bfefSLemover      val fake_ptw = Module(new FakePTW())
76892e3bfefSLemover      io <> fake_ptw.io
76992e3bfefSLemover      Seq()
77092e3bfefSLemover    }
77192e3bfefSLemover    else {
77292e3bfefSLemover        io <> ptw.module.io
77392e3bfefSLemover        ptw.module.getPerfEvents
77492e3bfefSLemover    }
77592e3bfefSLemover    generatePerfEvent()
77692e3bfefSLemover  }
777935edac4STang Haojin
778935edac4STang Haojin  lazy val module = new L2TLBWrapperImp(this)
77992e3bfefSLemover}
780