xref: /XiangShan/src/main/scala/xiangshan/cache/mmu/TLB.scala (revision e9027bcd3cfc82c9caf06d8f0c2b2ba74c4992a2)
16d5ddbceSLemover/***************************************************************************************
26d5ddbceSLemover* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
3f320e0f0SYinan Xu* Copyright (c) 2020-2021 Peng Cheng Laboratory
46d5ddbceSLemover*
56d5ddbceSLemover* XiangShan is licensed under Mulan PSL v2.
66d5ddbceSLemover* You can use this software according to the terms and conditions of the Mulan PSL v2.
76d5ddbceSLemover* You may obtain a copy of Mulan PSL v2 at:
86d5ddbceSLemover*          http://license.coscl.org.cn/MulanPSL2
9f1fe8698SLemover
106d5ddbceSLemover* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
116d5ddbceSLemover* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
126d5ddbceSLemover* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
136d5ddbceSLemover*
146d5ddbceSLemover* See the Mulan PSL v2 for more details.
156d5ddbceSLemover***************************************************************************************/
166d5ddbceSLemover
176d5ddbceSLemoverpackage xiangshan.cache.mmu
186d5ddbceSLemover
198891a219SYinan Xuimport org.chipsalliance.cde.config.Parameters
206d5ddbceSLemoverimport chisel3._
216d5ddbceSLemoverimport chisel3.util._
225ab1b84dSHaoyuan Fengimport difftest._
23a0301c0dSLemoverimport freechips.rocketchip.util.SRAMAnnotation
246d5ddbceSLemoverimport xiangshan._
256d5ddbceSLemoverimport utils._
263c02ee8fSwakafaimport utility._
27f1fe8698SLemoverimport xiangshan.backend.fu.{PMPChecker, PMPReqBundle, PMPConfig => XSPMPConfig}
289aca92b9SYinan Xuimport xiangshan.backend.rob.RobPtr
296d5ddbceSLemoverimport xiangshan.backend.fu.util.HasCSRConst
30f1fe8698SLemoverimport freechips.rocketchip.rocket.PMPConfig
316d5ddbceSLemover
32f1fe8698SLemover/** TLB module
33f1fe8698SLemover  * support block request and non-block request io at the same time
34f1fe8698SLemover  * return paddr at next cycle, then go for pmp/pma check
35f1fe8698SLemover  * @param Width: The number of requestors
36f1fe8698SLemover  * @param Block: Blocked or not for each requestor ports
37f1fe8698SLemover  * @param q: TLB Parameters, like entry number, each TLB has its own parameters
38f1fe8698SLemover  * @param p: XiangShan Paramemters, like XLEN
39f1fe8698SLemover  */
40a0301c0dSLemover
4103efd994Shappy-lxclass TLB(Width: Int, nRespDups: Int = 1, Block: Seq[Boolean], q: TLBParameters)(implicit p: Parameters) extends TlbModule
42f1fe8698SLemover  with HasCSRConst
43f1fe8698SLemover  with HasPerfEvents
44f1fe8698SLemover{
4503efd994Shappy-lx  val io = IO(new TlbIO(Width, nRespDups, q))
46a0301c0dSLemover
476d5ddbceSLemover  val req = io.requestor.map(_.req)
486d5ddbceSLemover  val resp = io.requestor.map(_.resp)
496d5ddbceSLemover  val ptw = io.ptw
50b6982e83SLemover  val pmp = io.pmp
518744445eSMaxpicca-Li  val refill_to_mem = io.refill_to_mem
526d5ddbceSLemover
53f1fe8698SLemover  /** Sfence.vma & Svinval
54f1fe8698SLemover    * Sfence.vma will 1. flush old entries 2. flush inflight 3. flush pipe
55f1fe8698SLemover    * Svinval will 1. flush old entries 2. flush inflight
56f1fe8698SLemover    * So, Svinval will not flush pipe, which means
57f1fe8698SLemover    * it should not drop reqs from pipe and should return right resp
58f1fe8698SLemover    */
59f1fe8698SLemover  val sfence = DelayN(io.sfence, q.fenceDelay)
606d5ddbceSLemover  val csr = io.csr
61f1fe8698SLemover  val satp = DelayN(io.csr.satp, q.fenceDelay)
62d0de7e4aSpeixiaokun  val vsatp = DelayN(io.csr.vsatp, q.fenceDelay)
63d0de7e4aSpeixiaokun  val hgatp = DelayN(io.csr.hgatp, q.fenceDelay)
64d0de7e4aSpeixiaokun
65d0de7e4aSpeixiaokun  val flush_mmu = DelayN(sfence.valid || csr.satp.changed || csr.vsatp.changed || csr.hgatp.changed, q.fenceDelay)
66f1fe8698SLemover  val mmu_flush_pipe = DelayN(sfence.valid && sfence.bits.flushPipe, q.fenceDelay) // for svinval, won't flush pipe
67f1fe8698SLemover  val flush_pipe = io.flushPipe
68f1fe8698SLemover
69ffa711ffSpeixiaokun  val req_in = req
70ffa711ffSpeixiaokun  val req_out = req.map(a => RegEnable(a.bits, a.fire()))
71ffa711ffSpeixiaokun  val req_out_v = (0 until Width).map(i => ValidHold(req_in(i).fire && !req_in(i).bits.kill, resp(i).fire, flush_pipe(i)))
72ffa711ffSpeixiaokun
73ffa711ffSpeixiaokun  val isHyperInst = (0 until Width).map(i => req_out_v(i) && req_out(i).hyperinst)
7450c7aa78Speixiaokun
75f1fe8698SLemover  // ATTENTION: csr and flush from backend are delayed. csr should not be later than flush.
76f1fe8698SLemover  // because, csr will influence tlb behavior.
77a0301c0dSLemover  val ifecth = if (q.fetchi) true.B else false.B
78d0de7e4aSpeixiaokun  val mode_tmp = if (q.useDmode) csr.priv.dmode else csr.priv.imode
79d0de7e4aSpeixiaokun  val mode = (0 until Width).map(i => Mux(isHyperInst(i), csr.priv.spvp, mode_tmp))
80d0de7e4aSpeixiaokun  val virt = csr.priv.virt
81d0de7e4aSpeixiaokun  val sum = (0 until Width).map(i => Mux(virt || isHyperInst(i), io.csr.priv.vsum, io.csr.priv.sum))
82d0de7e4aSpeixiaokun  val mxr = (0 until Width).map(i => Mux(virt || isHyperInst(i), io.csr.priv.vmxr || io.csr.priv.mxr, io.csr.priv.mxr))
83ffa711ffSpeixiaokun  val req_in_s2xlate = (0 until Width).map(i => MuxCase(noS2xlate, Seq(
84ffa711ffSpeixiaokun      (!(virt || req_in(i).bits.hyperinst)) -> noS2xlate,
85ffa711ffSpeixiaokun      (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage,
86ffa711ffSpeixiaokun      (vsatp.mode === 0.U) -> onlyStage2,
87ffa711ffSpeixiaokun      (hgatp.mode === 0.U) -> onlyStage1
88ffa711ffSpeixiaokun    )))
89ffa711ffSpeixiaokun  val req_out_s2xlate = (0 until Width).map(i => MuxCase(noS2xlate, Seq(
903106de0aSpeixiaokun    (!(virt || isHyperInst(i))) -> noS2xlate,
913106de0aSpeixiaokun    (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage,
923106de0aSpeixiaokun    (vsatp.mode === 0.U) -> onlyStage2,
933106de0aSpeixiaokun    (hgatp.mode === 0.U) -> onlyStage1
943106de0aSpeixiaokun  )))
95*e9027bcdSpeixiaokun  val need_gpa = RegInit(false.B)
96*e9027bcdSpeixiaokun  val need_gpa_vpn = Reg(UInt(vpnLen.W))
97*e9027bcdSpeixiaokun  val need_gpa_gvpn = Reg(UInt(vpnLen.W))
98*e9027bcdSpeixiaokun  val hasGpf = Wire(Vec(Width, Bool()))
99d0de7e4aSpeixiaokun
1006d5ddbceSLemover  // val vmEnable = satp.mode === 8.U // && (mode < ModeM) // FIXME: fix me when boot xv6/linux...
101d0de7e4aSpeixiaokun  val vmEnable = (0 until Width).map(i => if (EnbaleTlbDebug) (satp.mode === 8.U)
102d0de7e4aSpeixiaokun    else (satp.mode === 8.U) && (mode(i) < ModeM))
103d0de7e4aSpeixiaokun  val s2xlateEnable = (0 until Width).map(i => (isHyperInst(i) || virt) && (vsatp.mode === 8.U || hgatp.mode === 8.U) && (mode(i) < ModeM))
104d0de7e4aSpeixiaokun  val portTranslateEnable = (0 until Width).map(i => (vmEnable(i) || s2xlateEnable(i)) && RegNext(!req(i).bits.no_translate))
1056d5ddbceSLemover
1066d5ddbceSLemover
107*e9027bcdSpeixiaokun  val refill = (0 until Width).map(i => ptw.resp.fire && !(need_gpa && need_gpa_vpn === ptw.resp.bits.getVpn) && !flush_mmu && (vmEnable(i) || ptw.resp.bits.s2xlate =/= noS2xlate))
108eb4bf3f2Speixiaokun  refill_to_mem := DontCare
10903efd994Shappy-lx  val entries = Module(new TlbStorageWrapper(Width, q, nRespDups))
110f1fe8698SLemover  entries.io.base_connect(sfence, csr, satp)
111f1fe8698SLemover  if (q.outReplace) { io.replace <> entries.io.replace }
1126d5ddbceSLemover  for (i <- 0 until Width) {
113ffa711ffSpeixiaokun    entries.io.r_req_apply(io.requestor(i).req.valid, get_pn(req_in(i).bits.vaddr), i, req_in_s2xlate(i))
114cca17e78Speixiaokun    entries.io.w_apply(refill(i), ptw.resp.bits)
1158744445eSMaxpicca-Li    resp(i).bits.debug.isFirstIssue := RegNext(req(i).bits.debug.isFirstIssue)
1168744445eSMaxpicca-Li    resp(i).bits.debug.robIdx := RegNext(req(i).bits.debug.robIdx)
117a0301c0dSLemover  }
118*e9027bcdSpeixiaokun
119f1fe8698SLemover  // read TLB, get hit/miss, paddr, perm bits
120f1fe8698SLemover  val readResult = (0 until Width).map(TLBRead(_))
121f1fe8698SLemover  val hitVec = readResult.map(_._1)
122f1fe8698SLemover  val missVec = readResult.map(_._2)
123f1fe8698SLemover  val pmp_addr = readResult.map(_._3)
124f9ac118cSHaoyuan Feng  val perm = readResult.map(_._4)
1253106de0aSpeixiaokun  val g_perm = readResult.map(_._5)
126f1fe8698SLemover  // check pmp use paddr (for timing optization, use pmp_addr here)
127f1fe8698SLemover  // check permisson
128f1fe8698SLemover  (0 until Width).foreach{i =>
129f1fe8698SLemover    pmp_check(pmp_addr(i), req_out(i).size, req_out(i).cmd, i)
13003efd994Shappy-lx    for (d <- 0 until nRespDups) {
131ffa711ffSpeixiaokun      perm_check(perm(i)(d), req_out(i).cmd, i, d, g_perm(i)(d), req_out(i).hlvx, req_out_s2xlate(i))
13203efd994Shappy-lx    }
133c3d5cfb3Speixiaokun    hasGpf(i) := resp(i).bits.excp(0).gpf.ld || resp(i).bits.excp(0).gpf.st || resp(i).bits.excp(0).gpf.instr
134f1fe8698SLemover  }
1356d5ddbceSLemover
136f1fe8698SLemover  // handle block or non-block io
137f1fe8698SLemover  // for non-block io, just return the above result, send miss to ptw
138f1fe8698SLemover  // for block io, hold the request, send miss to ptw,
139f1fe8698SLemover  //   when ptw back, return the result
140f1fe8698SLemover  (0 until Width) foreach {i =>
141f1fe8698SLemover    if (Block(i)) handle_block(i)
142f1fe8698SLemover    else handle_nonblock(i)
143f1fe8698SLemover  }
144f1fe8698SLemover  io.ptw.resp.ready := true.B
145a0301c0dSLemover
146f1fe8698SLemover  /************************  main body above | method/log/perf below ****************************/
147f1fe8698SLemover  def TLBRead(i: Int) = {
148ffa711ffSpeixiaokun    val (e_hit, e_ppn, e_perm, e_g_perm, e_s2xlate) = entries.io.r_resp_apply(i)
149ffa711ffSpeixiaokun    val (p_hit, p_ppn, p_perm, p_gvpn, p_g_perm, p_s2xlate) = ptw_resp_bypass(get_pn(req_in(i).bits.vaddr), req_in_s2xlate(i))
150292bea3fSWilliam Wang    val enable = portTranslateEnable(i)
151f86480a7Speixiaokun    val isOnlys2xlate = req_out_s2xlate(i) === onlyStage2
1523106de0aSpeixiaokun    val resp_gpa_refill = RegInit(false.B)
1533106de0aSpeixiaokun    val need_gpa_vpn_hit = RegEnable(need_gpa_vpn === get_pn(req_in(i).bits.vaddr), req_in(i).fire())
154f86480a7Speixiaokun    when (io.requestor(i).resp.valid && hasGpf(i) && need_gpa === false.B && !need_gpa_vpn_hit && !isOnlys2xlate) {
155c3d5cfb3Speixiaokun      need_gpa := true.B
1563106de0aSpeixiaokun      need_gpa_vpn := get_pn(req_out(i).vaddr)
1573106de0aSpeixiaokun      resp_gpa_refill := false.B
158c3d5cfb3Speixiaokun    }
1593106de0aSpeixiaokun    when (ptw.resp.fire && need_gpa && need_gpa_vpn === ptw.resp.bits.getVpn) {
1603106de0aSpeixiaokun      need_gpa_gvpn := Mux(ptw.resp.bits.s2xlate === onlyStage2, Cat(ptw.resp.bits.s1.entry.tag, ptw.resp.bits.s1.ppn_low(OHToUInt(ptw.resp.bits.s1.pteidx))), ptw.resp.bits.s2.entry.tag)
1613106de0aSpeixiaokun      resp_gpa_refill := true.B
1623106de0aSpeixiaokun    }
1633106de0aSpeixiaokun
164933ec998Speixiaokun    when (hasGpf(i) && resp_gpa_refill && need_gpa_vpn_hit){
165c3d5cfb3Speixiaokun      need_gpa := false.B
166c3d5cfb3Speixiaokun    }
167c3d5cfb3Speixiaokun
168cb8f2f2aSLemover    val hit = e_hit || p_hit
169f86480a7Speixiaokun    val miss = (!hit && enable) || hasGpf(i) && !(resp_gpa_refill && need_gpa_vpn_hit) && !isOnlys2xlate
170f1fe8698SLemover    hit.suggestName(s"hit_read_${i}")
171f1fe8698SLemover    miss.suggestName(s"miss_read_${i}")
1726d5ddbceSLemover
173f1fe8698SLemover    val vaddr = SignExt(req_out(i).vaddr, PAddrBits)
174f1fe8698SLemover    resp(i).bits.miss := miss
175935edac4STang Haojin    resp(i).bits.ptwBack := ptw.resp.fire
1768744445eSMaxpicca-Li    resp(i).bits.memidx := RegNext(req_in(i).bits.memidx)
1776d5ddbceSLemover
17803efd994Shappy-lx    val ppn = WireInit(VecInit(Seq.fill(nRespDups)(0.U(ppnLen.W))))
17903efd994Shappy-lx    val perm = WireInit(VecInit(Seq.fill(nRespDups)(0.U.asTypeOf(new TlbPermBundle))))
18082978df9Speixiaokun    val gvpn = WireInit(VecInit(Seq.fill(nRespDups)(0.U(vpnLen.W))))
181d0de7e4aSpeixiaokun    val g_perm = WireInit(VecInit(Seq.fill(nRespDups)(0.U.asTypeOf(new TlbPermBundle))))
18250c7aa78Speixiaokun    val r_s2xlate = WireInit(VecInit(Seq.fill(nRespDups)(0.U(2.W))))
18303efd994Shappy-lx    for (d <- 0 until nRespDups) {
18403efd994Shappy-lx      ppn(d) := Mux(p_hit, p_ppn, e_ppn(d))
18503efd994Shappy-lx      perm(d) := Mux(p_hit, p_perm, e_perm(d))
186c794d992Speixiaokun      gvpn(d) :=  Mux(hasGpf(i), need_gpa_gvpn, 0.U)
187d0de7e4aSpeixiaokun      g_perm(d) := Mux(p_hit, p_g_perm, e_g_perm(d))
18850c7aa78Speixiaokun      r_s2xlate(d) := Mux(p_hit, p_s2xlate, e_s2xlate(d))
18903efd994Shappy-lx      val paddr = Cat(ppn(d), get_off(req_out(i).vaddr))
190d0de7e4aSpeixiaokun      val gpaddr = Cat(gvpn(d), get_off(req_out(i).vaddr))
191292bea3fSWilliam Wang      resp(i).bits.paddr(d) := Mux(enable, paddr, vaddr)
19250c7aa78Speixiaokun      resp(i).bits.gpaddr(d) := Mux(r_s2xlate(d) === onlyStage2, vaddr, gpaddr)
19303efd994Shappy-lx    }
19403efd994Shappy-lx
19503efd994Shappy-lx    XSDebug(req_out_v(i), p"(${i.U}) hit:${hit} miss:${miss} ppn:${Hexadecimal(ppn(0))} perm:${perm(0)}\n")
19603efd994Shappy-lx
197f9ac118cSHaoyuan Feng    val pmp_paddr = resp(i).bits.paddr(0)
198f1fe8698SLemover
1993106de0aSpeixiaokun    (hit, miss, pmp_paddr, perm, g_perm)
200f1fe8698SLemover  }
201f1fe8698SLemover
202f1fe8698SLemover  def pmp_check(addr: UInt, size: UInt, cmd: UInt, idx: Int): Unit = {
203f1fe8698SLemover    pmp(idx).valid := resp(idx).valid
204f1fe8698SLemover    pmp(idx).bits.addr := addr
205f1fe8698SLemover    pmp(idx).bits.size := size
206f1fe8698SLemover    pmp(idx).bits.cmd := cmd
207f1fe8698SLemover  }
208f1fe8698SLemover
209d0de7e4aSpeixiaokun  def perm_check(perm: TlbPermBundle, cmd: UInt, idx: Int, nDups: Int, g_perm: TlbPermBundle, hlvx: Bool, s2xlate: UInt) = {
2105b7ef044SLemover    // for timing optimization, pmp check is divided into dynamic and static
2115b7ef044SLemover    // dynamic: superpage (or full-connected reg entries) -> check pmp when translation done
2125b7ef044SLemover    // static: 4K pages (or sram entries) -> check pmp with pre-checked results
213c3d5cfb3Speixiaokun    val hasS2xlate = s2xlate =/= noS2xlate
21407f77bf0Speixiaokun    val onlyS2 = s2xlate === onlyStage2
215d0de7e4aSpeixiaokun    val af = perm.af || (hasS2xlate && g_perm.af)
216d0de7e4aSpeixiaokun
217d0de7e4aSpeixiaokun    // Stage 1 perm check
218382a2ebdSpeixiaokun    val pf = perm.pf || (hlvx && !perm.x)
219f1fe8698SLemover    val ldUpdate = !perm.a && TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd) // update A/D through exception
220f1fe8698SLemover    val stUpdate = (!perm.a || !perm.d) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd)) // update A/D through exception
221f1fe8698SLemover    val instrUpdate = !perm.a && TlbCmd.isExec(cmd) // update A/D through exception
222d0de7e4aSpeixiaokun    val modeCheck = !(mode(idx) === ModeU && !perm.u || mode(idx) === ModeS && perm.u && (!sum(idx) || ifecth))
223d0de7e4aSpeixiaokun    val ldPermFail = !(modeCheck && (perm.r || mxr(idx) && perm.x))
224a79fef67Swakafa    val stPermFail = !(modeCheck && perm.w)
225a79fef67Swakafa    val instrPermFail = !(modeCheck && perm.x)
226f1fe8698SLemover    val ldPf = (ldPermFail || pf) && (TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd))
227f1fe8698SLemover    val stPf = (stPermFail || pf) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd))
228f1fe8698SLemover    val instrPf = (instrPermFail || pf) && TlbCmd.isExec(cmd)
229d0de7e4aSpeixiaokun    val s1_valid = portTranslateEnable(idx) && !onlyS2
230d0de7e4aSpeixiaokun
231d0de7e4aSpeixiaokun    // Stage 2 perm check
232382a2ebdSpeixiaokun    val gpf = g_perm.pf || (hlvx && !g_perm.x)
233d0de7e4aSpeixiaokun    val g_ldUpdate = !g_perm.a && TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd)
234d0de7e4aSpeixiaokun    val g_stUpdate = (!g_perm.a || !g_perm.d) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd))
235d0de7e4aSpeixiaokun    val g_instrUpdate = !g_perm.a && TlbCmd.isExec(cmd)
236d0de7e4aSpeixiaokun    val g_ldPermFail = !(g_perm.r || io.csr.priv.mxr && g_perm.x)
237d0de7e4aSpeixiaokun    val g_stPermFail = !g_perm.w
238d0de7e4aSpeixiaokun    val g_instrPermFail = !g_perm.x
239d0de7e4aSpeixiaokun    val ldGpf = (g_ldPermFail || gpf) && (TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd))
240d0de7e4aSpeixiaokun    val stGpf = (g_stPermFail || gpf) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd))
241d0de7e4aSpeixiaokun    val instrGpf = (g_instrPermFail || gpf) && TlbCmd.isExec(cmd)
24244f8e3e4Speixiaokun    val s2_valid = hasS2xlate && portTranslateEnable(idx)
243d0de7e4aSpeixiaokun
244d0de7e4aSpeixiaokun    val fault_valid = s1_valid || s2_valid
245d0de7e4aSpeixiaokun
246c794d992Speixiaokun    // when pf and gpf can't happens simultaneously
247c794d992Speixiaokun    val hasPf = (ldPf || ldUpdate || stPf || stUpdate || instrPf || instrUpdate) && s1_valid && !af
248d0de7e4aSpeixiaokun    resp(idx).bits.excp(nDups).pf.ld := (ldPf || ldUpdate) && s1_valid && !af
249d0de7e4aSpeixiaokun    resp(idx).bits.excp(nDups).pf.st := (stPf || stUpdate) && s1_valid && !af
250d0de7e4aSpeixiaokun    resp(idx).bits.excp(nDups).pf.instr := (instrPf || instrUpdate) && s1_valid && !af
251b6982e83SLemover    // NOTE: pf need && with !af, page fault has higher priority than access fault
252b6982e83SLemover    // but ptw may also have access fault, then af happens, the translation is wrong.
253b6982e83SLemover    // In this case, pf has lower priority than af
2546d5ddbceSLemover
255c794d992Speixiaokun    resp(idx).bits.excp(nDups).gpf.ld := (ldGpf || g_ldUpdate) && s2_valid && !af && !hasPf
256c794d992Speixiaokun    resp(idx).bits.excp(nDups).gpf.st := (stGpf || g_stUpdate) && s2_valid && !af && !hasPf
257c794d992Speixiaokun    resp(idx).bits.excp(nDups).gpf.instr := (instrGpf || g_instrUpdate) && s2_valid && !af && !hasPf
258d0de7e4aSpeixiaokun
259f9ac118cSHaoyuan Feng    resp(idx).bits.excp(nDups).af.ld    := af && TlbCmd.isRead(cmd) && fault_valid
260f9ac118cSHaoyuan Feng    resp(idx).bits.excp(nDups).af.st    := af && TlbCmd.isWrite(cmd) && fault_valid
261f9ac118cSHaoyuan Feng    resp(idx).bits.excp(nDups).af.instr := af && TlbCmd.isExec(cmd) && fault_valid
262d0de7e4aSpeixiaokun
263d0de7e4aSpeixiaokun
2646d5ddbceSLemover  }
2656d5ddbceSLemover
266f1fe8698SLemover  def handle_nonblock(idx: Int): Unit = {
267f1fe8698SLemover    io.requestor(idx).resp.valid := req_out_v(idx)
268f1fe8698SLemover    io.requestor(idx).req.ready := io.requestor(idx).resp.ready // should always be true
2699930e66fSLemover    XSError(!io.requestor(idx).resp.ready, s"${q.name} port ${idx} is non-block, resp.ready must be true.B")
270cb8f2f2aSLemover
271c3d5cfb3Speixiaokun    val req_need_gpa = hasGpf(idx)
272d0de7e4aSpeixiaokun    val req_s2xlate = Wire(UInt(2.W))
27382978df9Speixiaokun    req_s2xlate := MuxCase(noS2xlate, Seq(
274496c751cSpeixiaokun      (!(virt || req_out(idx).hyperinst)) -> noS2xlate,
275cca17e78Speixiaokun      (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage,
276cca17e78Speixiaokun      (vsatp.mode === 0.U) -> onlyStage2,
277c3d5cfb3Speixiaokun      (hgatp.mode === 0.U || req_need_gpa) -> onlyStage1
27882978df9Speixiaokun    ))
279933ec998Speixiaokun    val ptw_s2xlate = ptw.resp.bits.s2xlate
280933ec998Speixiaokun    val has_s2xlate = ptw_s2xlate =/= noS2xlate
28182978df9Speixiaokun    val onlyS2 = ptw_s2xlate === onlyStage2
282933ec998Speixiaokun    val ptw_s1_hit = ptw.resp.bits.s1.hit(get_pn(req_out(idx).vaddr), Mux(has_s2xlate, io.csr.vsatp.asid, io.csr.satp.asid), io.csr.hgatp.asid, true, false, has_s2xlate)
283d0de7e4aSpeixiaokun    val ptw_s2_hit = ptw.resp.bits.s2.hit(get_pn(req_out(idx).vaddr), io.csr.hgatp.asid)
284d0de7e4aSpeixiaokun    val ptw_just_back = ptw.resp.fire && req_s2xlate === ptw_s2xlate && Mux(onlyS2, ptw_s2_hit, ptw_s1_hit)
285185e6164SHaoyuan Feng    val ptw_already_back = RegNext(ptw.resp.fire) && RegNext(ptw.resp.bits).hit(get_pn(req_out(idx).vaddr), asid = io.csr.satp.asid, allType = true)
286c3d5cfb3Speixiaokun    io.ptw.req(idx).valid := req_out_v(idx) && (missVec(idx)) && !(ptw_just_back || ptw_already_back) // TODO: remove the regnext, timing
287185e6164SHaoyuan Feng    io.tlbreplay(idx) := req_out_v(idx) && missVec(idx) && (ptw_just_back || ptw_already_back)
288185e6164SHaoyuan Feng    when (io.requestor(idx).req_kill && RegNext(io.requestor(idx).req.fire)) {
289c3b763d0SYinan Xu      io.ptw.req(idx).valid := false.B
290185e6164SHaoyuan Feng      io.tlbreplay(idx) := true.B
291c3b763d0SYinan Xu    }
292185e6164SHaoyuan Feng    io.ptw.req(idx).bits.vpn := get_pn(req_out(idx).vaddr)
293d0de7e4aSpeixiaokun    io.ptw.req(idx).bits.s2xlate := RegNext(req_s2xlate)
294185e6164SHaoyuan Feng    io.ptw.req(idx).bits.memidx := req_out(idx).memidx
295149086eaSLemover  }
296a0301c0dSLemover
297f1fe8698SLemover  def handle_block(idx: Int): Unit = {
298f1fe8698SLemover    // three valid: 1.if exist a entry; 2.if sent to ptw; 3.unset resp.valid
299935edac4STang Haojin    io.requestor(idx).req.ready := !req_out_v(idx) || io.requestor(idx).resp.fire
300f1fe8698SLemover    // req_out_v for if there is a request, may long latency, fixme
301f1fe8698SLemover
302f1fe8698SLemover    // miss request entries
303c3d5cfb3Speixiaokun    val req_need_gpa = hasGpf(idx)
304f1fe8698SLemover    val miss_req_vpn = get_pn(req_out(idx).vaddr)
3058744445eSMaxpicca-Li    val miss_req_memidx = req_out(idx).memidx
306d0de7e4aSpeixiaokun    val miss_req_s2xlate = Wire(UInt(2.W))
30782978df9Speixiaokun    miss_req_s2xlate := MuxCase(noS2xlate, Seq(
308496c751cSpeixiaokun      (!(virt || req_out(idx).hyperinst)) -> noS2xlate,
309cca17e78Speixiaokun      (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage,
310cca17e78Speixiaokun      (vsatp.mode === 0.U) -> onlyStage2,
311c3d5cfb3Speixiaokun      (hgatp.mode === 0.U || req_need_gpa) -> onlyStage1
31282978df9Speixiaokun    ))
313c3d5cfb3Speixiaokun    val miss_req_s2xlate_reg = RegEnable(miss_req_s2xlate, io.ptw.req(idx).fire())
314c3d5cfb3Speixiaokun    val hasS2xlate = miss_req_s2xlate_reg =/= noS2xlate
315c3d5cfb3Speixiaokun    val onlyS2 = miss_req_s2xlate_reg === onlyStage2
31682978df9Speixiaokun    val hit_s1 = io.ptw.resp.bits.s1.hit(miss_req_vpn, Mux(hasS2xlate, io.csr.vsatp.asid, io.csr.satp.asid), io.csr.hgatp.asid, allType = true, false, hasS2xlate)
317d0de7e4aSpeixiaokun    val hit_s2 = io.ptw.resp.bits.s2.hit(miss_req_vpn, io.csr.hgatp.asid)
318c3d5cfb3Speixiaokun    val hit = Mux(onlyS2, hit_s2, hit_s1) && io.ptw.resp.valid && miss_req_s2xlate_reg === io.ptw.resp.bits.s2xlate
319f1fe8698SLemover
320f1fe8698SLemover    val new_coming = RegNext(req_in(idx).fire && !req_in(idx).bits.kill && !flush_pipe(idx), false.B)
321f1fe8698SLemover    val miss_wire = new_coming && missVec(idx)
322935edac4STang Haojin    val miss_v = ValidHoldBypass(miss_wire, resp(idx).fire, flush_pipe(idx))
323f1fe8698SLemover    val miss_req_v = ValidHoldBypass(miss_wire || (miss_v && flush_mmu && !mmu_flush_pipe),
324935edac4STang Haojin      io.ptw.req(idx).fire || resp(idx).fire, flush_pipe(idx))
325f1fe8698SLemover
326f1fe8698SLemover    // when ptw resp, check if hit, reset miss_v, resp to lsu/ifu
327292bea3fSWilliam Wang    resp(idx).valid := req_out_v(idx) && !(miss_v && portTranslateEnable(idx))
328935edac4STang Haojin    when (io.ptw.resp.fire && hit && req_out_v(idx) && portTranslateEnable(idx)) {
329d0de7e4aSpeixiaokun      val stage1 = io.ptw.resp.bits.s1
330d0de7e4aSpeixiaokun      val stage2 = io.ptw.resp.bits.s2
331d0de7e4aSpeixiaokun      val s2xlate = io.ptw.resp.bits.s2xlate
332f1fe8698SLemover      resp(idx).valid := true.B
333c3d5cfb3Speixiaokun      resp(idx).bits.miss := false.B
334d0de7e4aSpeixiaokun      val s1_paddr = Cat(stage1.genPPN(get_pn(req_out(idx).vaddr)), get_off(req_out(idx).vaddr))
3357e664aa3Speixiaokun      val s2_paddr = Cat(stage2.genPPNS2(get_pn(req_out(idx).vaddr)), get_off(req_out(idx).vaddr))
33603efd994Shappy-lx      for (d <- 0 until nRespDups) {
33782978df9Speixiaokun        resp(idx).bits.paddr(d) := Mux(s2xlate =/= noS2xlate, s2_paddr, s1_paddr)
338d0de7e4aSpeixiaokun        resp(idx).bits.gpaddr(d) := s1_paddr
339cca17e78Speixiaokun        perm_check(stage1, req_out(idx).cmd, idx, d, stage2, req_out(idx).hlvx, s2xlate)
34003efd994Shappy-lx      }
34103efd994Shappy-lx      pmp_check(resp(idx).bits.paddr(0), req_out(idx).size, req_out(idx).cmd, idx)
342f1fe8698SLemover
343f1fe8698SLemover      // NOTE: the unfiltered req would be handled by Repeater
344f1fe8698SLemover    }
345f1fe8698SLemover    assert(RegNext(!resp(idx).valid || resp(idx).ready, true.B), "when tlb resp valid, ready should be true, must")
346f1fe8698SLemover    assert(RegNext(req_out_v(idx) || !(miss_v || miss_req_v), true.B), "when not req_out_v, should not set miss_v/miss_req_v")
347f1fe8698SLemover
348f1fe8698SLemover    val ptw_req = io.ptw.req(idx)
349f1fe8698SLemover    ptw_req.valid := miss_req_v
350f1fe8698SLemover    ptw_req.bits.vpn := miss_req_vpn
351d0de7e4aSpeixiaokun    ptw_req.bits.s2xlate := miss_req_s2xlate
3528744445eSMaxpicca-Li    ptw_req.bits.memidx := miss_req_memidx
353f1fe8698SLemover
354185e6164SHaoyuan Feng    io.tlbreplay(idx) := false.B
355185e6164SHaoyuan Feng
356f1fe8698SLemover    // NOTE: when flush pipe, tlb should abandon last req
357f1fe8698SLemover    // however, some outside modules like icache, dont care flushPipe, and still waiting for tlb resp
358f1fe8698SLemover    // just resp valid and raise page fault to go through. The pipe(ifu) will abandon it.
359f1fe8698SLemover    if (!q.outsideRecvFlush) {
360292bea3fSWilliam Wang      when (req_out_v(idx) && flush_pipe(idx) && portTranslateEnable(idx)) {
361f1fe8698SLemover        resp(idx).valid := true.B
36203efd994Shappy-lx        for (d <- 0 until nRespDups) {
36303efd994Shappy-lx          resp(idx).bits.excp(d).pf.ld := true.B // sfence happened, pf for not to use this addr
36403efd994Shappy-lx          resp(idx).bits.excp(d).pf.st := true.B
36503efd994Shappy-lx          resp(idx).bits.excp(d).pf.instr := true.B
36603efd994Shappy-lx        }
367f1fe8698SLemover      }
368f1fe8698SLemover    }
369f1fe8698SLemover  }
370cb8f2f2aSLemover
371cb8f2f2aSLemover  // when ptw resp, tlb at refill_idx maybe set to miss by force.
372cb8f2f2aSLemover  // Bypass ptw resp to check.
373d0de7e4aSpeixiaokun  def ptw_resp_bypass(vpn: UInt, s2xlate: UInt) = {
374cca17e78Speixiaokun    val hasS2xlate = s2xlate =/= noS2xlate
375cca17e78Speixiaokun    val onlyS2 = s2xlate === onlyStage2
376c3d5cfb3Speixiaokun    val onlyS1 = s2xlate === onlyStage1
377d0de7e4aSpeixiaokun    val s2xlate_hit = s2xlate === ptw.resp.bits.s2xlate
378d0de7e4aSpeixiaokun    val normal_hit = ptw.resp.bits.s1.hit(vpn, Mux(hasS2xlate, io.csr.vsatp.asid, io.csr.satp.asid), io.csr.hgatp.asid, true, false, hasS2xlate)
379d0de7e4aSpeixiaokun    val onlyS2_hit = ptw.resp.bits.s2.hit(vpn, io.csr.hgatp.asid)
380d0de7e4aSpeixiaokun    val p_hit = RegNext(Mux(onlyS2, onlyS2_hit, normal_hit) && io.ptw.resp.fire && s2xlate_hit)
381d0de7e4aSpeixiaokun    val ppn_s1 = ptw.resp.bits.s1.genPPN(vpn)
3827e664aa3Speixiaokun    val ppn_s2 = ptw.resp.bits.s2.genPPNS2(vpn)
383d0de7e4aSpeixiaokun    val p_ppn = RegEnable(Mux(hasS2xlate, ppn_s2, ppn_s1), io.ptw.resp.fire)
384d0de7e4aSpeixiaokun    val p_perm = RegEnable(ptwresp_to_tlbperm(ptw.resp.bits.s1), io.ptw.resp.fire)
385c3d5cfb3Speixiaokun    val p_gvpn = RegEnable(Mux(onlyS1, Cat(ptw.resp.bits.s1.entry.tag, ptw.resp.bits.s1.ppn_low(OHToUInt(ptw.resp.bits.s1.pteidx))), ptw.resp.bits.s2.entry.tag), io.ptw.resp.fire)
386d0de7e4aSpeixiaokun    val p_g_perm = RegEnable(hptwresp_to_tlbperm(ptw.resp.bits.s2), io.ptw.resp.fire)
387d0de7e4aSpeixiaokun    val p_s2xlate = RegEnable(ptw.resp.bits.s2xlate, io.ptw.resp.fire)
388d0de7e4aSpeixiaokun    (p_hit, p_ppn, p_perm, p_gvpn, p_g_perm, p_s2xlate)
389cb8f2f2aSLemover  }
390cb8f2f2aSLemover
391f1fe8698SLemover  // assert
392f1fe8698SLemover  for(i <- 0 until Width) {
393f1fe8698SLemover    TimeOutAssert(req_out_v(i) && !resp(i).valid, timeOutThreshold, s"{q.name} port{i} long time no resp valid.")
394149086eaSLemover  }
395a0301c0dSLemover
396f1fe8698SLemover  // perf event
397935edac4STang Haojin  val result_ok = req_in.map(a => RegNext(a.fire))
398f1fe8698SLemover  val perfEvents =
399f1fe8698SLemover    Seq(
400935edac4STang Haojin      ("access", PopCount((0 until Width).map{i => if (Block(i)) io.requestor(i).req.fire else portTranslateEnable(i) && result_ok(i) })),
401935edac4STang Haojin      ("miss  ", PopCount((0 until Width).map{i => if (Block(i)) portTranslateEnable(i) && result_ok(i) && missVec(i) else ptw.req(i).fire })),
402a0301c0dSLemover    )
403f1fe8698SLemover  generatePerfEvent()
404a0301c0dSLemover
405f1fe8698SLemover  // perf log
4066d5ddbceSLemover  for (i <- 0 until Width) {
407f1fe8698SLemover    if (Block(i)) {
408292bea3fSWilliam Wang      XSPerfAccumulate(s"access${i}",result_ok(i) && portTranslateEnable(i))
409f1fe8698SLemover      XSPerfAccumulate(s"miss${i}", result_ok(i) && missVec(i))
4106d5ddbceSLemover    } else {
411292bea3fSWilliam Wang      XSPerfAccumulate("first_access" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && RegNext(req(i).bits.debug.isFirstIssue))
412292bea3fSWilliam Wang      XSPerfAccumulate("access" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i))
413292bea3fSWilliam Wang      XSPerfAccumulate("first_miss" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && missVec(i) && RegNext(req(i).bits.debug.isFirstIssue))
414292bea3fSWilliam Wang      XSPerfAccumulate("miss" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && missVec(i))
415a0301c0dSLemover    }
4166d5ddbceSLemover  }
417935edac4STang Haojin  XSPerfAccumulate("ptw_resp_count", ptw.resp.fire)
418cca17e78Speixiaokun  XSPerfAccumulate("ptw_resp_pf_count", ptw.resp.fire && ptw.resp.bits.s1.pf)
4196d5ddbceSLemover
4206d5ddbceSLemover  // Log
4216d5ddbceSLemover  for(i <- 0 until Width) {
4226d5ddbceSLemover    XSDebug(req(i).valid, p"req(${i.U}): (${req(i).valid} ${req(i).ready}) ${req(i).bits}\n")
4236d5ddbceSLemover    XSDebug(resp(i).valid, p"resp(${i.U}): (${resp(i).valid} ${resp(i).ready}) ${resp(i).bits}\n")
4246d5ddbceSLemover  }
4256d5ddbceSLemover
426f1fe8698SLemover  XSDebug(io.sfence.valid, p"Sfence: ${io.sfence}\n")
427f1fe8698SLemover  XSDebug(ParallelOR(req_out_v) || ptw.resp.valid, p"vmEnable:${vmEnable} hit:${Binary(VecInit(hitVec).asUInt)} miss:${Binary(VecInit(missVec).asUInt)}\n")
4286d5ddbceSLemover  for (i <- ptw.req.indices) {
429935edac4STang Haojin    XSDebug(ptw.req(i).fire, p"L2TLB req:${ptw.req(i).bits}\n")
4306d5ddbceSLemover  }
43192e3bfefSLemover  XSDebug(ptw.resp.valid, p"L2TLB resp:${ptw.resp.bits} (v:${ptw.resp.valid}r:${ptw.resp.ready}) \n")
4326d5ddbceSLemover
433f9ac118cSHaoyuan Feng  println(s"${q.name}: page: ${q.NWays} ${q.Associative} ${q.Replacer.get}")
434a0301c0dSLemover
4355ab1b84dSHaoyuan Feng  if (env.EnableDifftest) {
4365ab1b84dSHaoyuan Feng    for (i <- 0 until Width) {
4375ab1b84dSHaoyuan Feng      val pf = io.requestor(i).resp.bits.excp(0).pf.instr || io.requestor(i).resp.bits.excp(0).pf.st || io.requestor(i).resp.bits.excp(0).pf.ld
438d0de7e4aSpeixiaokun      val gpf = io.requestor(i).resp.bits.excp(0).gpf.instr || io.requestor(i).resp.bits.excp(0).gpf.st || io.requestor(i).resp.bits.excp(0).gpf.ld
4395ab1b84dSHaoyuan Feng      val af = io.requestor(i).resp.bits.excp(0).af.instr || io.requestor(i).resp.bits.excp(0).af.st || io.requestor(i).resp.bits.excp(0).af.ld
4407d45a146SYinan Xu      val difftest = DifftestModule(new DiffL1TLBEvent)
441254e4960SHaoyuan Feng      difftest.coreid := io.hartId
442d0de7e4aSpeixiaokun      difftest.valid := RegNext(io.requestor(i).req.fire) && !io.requestor(i).req_kill && io.requestor(i).resp.fire && !io.requestor(i).resp.bits.miss && !pf && !af && !gpf && portTranslateEnable(i)
4437d45a146SYinan Xu      if (!Seq("itlb", "ldtlb", "sttlb").contains(q.name)) {
4447d45a146SYinan Xu        difftest.valid := false.B
4457d45a146SYinan Xu      }
4467d45a146SYinan Xu      difftest.index := TLBDiffId(p(XSCoreParamsKey).HartId).U
447d0de7e4aSpeixiaokun      difftest.satp := io.csr.satp
4487d45a146SYinan Xu      difftest.vpn := RegNext(get_pn(req_in(i).bits.vaddr))
4497d45a146SYinan Xu      difftest.ppn := get_pn(io.requestor(i).resp.bits.paddr(0))
45087d0ba30Speixiaokun      difftest.satp := Cat(io.csr.satp.mode, io.csr.satp.asid, io.csr.satp.ppn)
45187d0ba30Speixiaokun      difftest.vsatp := Cat(io.csr.vsatp.mode, io.csr.vsatp.asid, io.csr.vsatp.ppn)
45287d0ba30Speixiaokun      difftest.hgatp := Cat(io.csr.hgatp.mode, io.csr.hgatp.asid, io.csr.hgatp.ppn)
453dd103903Speixiaokun      val req_need_gpa = gpf
454dd103903Speixiaokun      val req_s2xlate = Wire(UInt(2.W))
455dd103903Speixiaokun      req_s2xlate := MuxCase(noS2xlate, Seq(
456dd103903Speixiaokun        (!(virt || RegNext(req_in(i).bits.hyperinst))) -> noS2xlate,
457cca17e78Speixiaokun        (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage,
458cca17e78Speixiaokun        (vsatp.mode === 0.U) -> onlyStage2,
459dd103903Speixiaokun        (hgatp.mode === 0.U || req_need_gpa) -> onlyStage1
46082978df9Speixiaokun      ))
461dd103903Speixiaokun      difftest.s2xlate := req_s2xlate
4627d45a146SYinan Xu    }
4635ab1b84dSHaoyuan Feng  }
4645ab1b84dSHaoyuan Feng}
4655ab1b84dSHaoyuan Feng
4667d45a146SYinan Xuobject TLBDiffId {
4677d45a146SYinan Xu  var i: Int = 0
4687d45a146SYinan Xu  var lastHartId: Int = -1
4697d45a146SYinan Xu  def apply(hartId: Int): Int = {
4707d45a146SYinan Xu    if (lastHartId != hartId) {
4717d45a146SYinan Xu      i = 0
4727d45a146SYinan Xu      lastHartId = hartId
4737d45a146SYinan Xu    }
4747d45a146SYinan Xu    i += 1
4757d45a146SYinan Xu    i - 1
4767d45a146SYinan Xu  }
477f1fe8698SLemover}
4781ca0e4f3SYinan Xu
47903efd994Shappy-lxclass TLBNonBlock(Width: Int, nRespDups: Int = 1, q: TLBParameters)(implicit p: Parameters) extends TLB(Width, nRespDups, Seq.fill(Width)(false), q)
48003efd994Shappy-lxclass TLBBLock(Width: Int, nRespDups: Int = 1, q: TLBParameters)(implicit p: Parameters) extends TLB(Width, nRespDups, Seq.fill(Width)(true), q)
4816d5ddbceSLemover
482a0301c0dSLemoverclass TlbReplace(Width: Int, q: TLBParameters)(implicit p: Parameters) extends TlbModule {
483a0301c0dSLemover  val io = IO(new TlbReplaceIO(Width, q))
484a0301c0dSLemover
485f9ac118cSHaoyuan Feng  if (q.Associative == "fa") {
486f9ac118cSHaoyuan Feng    val re = ReplacementPolicy.fromString(q.Replacer, q.NWays)
487f9ac118cSHaoyuan Feng    re.access(io.page.access.map(_.touch_ways))
488f9ac118cSHaoyuan Feng    io.page.refillIdx := re.way
489a0301c0dSLemover  } else { // set-acco && plru
490f9ac118cSHaoyuan Feng    val re = ReplacementPolicy.fromString(q.Replacer, q.NSets, q.NWays)
491f9ac118cSHaoyuan Feng    re.access(io.page.access.map(_.sets), io.page.access.map(_.touch_ways))
492f9ac118cSHaoyuan Feng    io.page.refillIdx := { if (q.NWays == 1) 0.U else re.way(io.page.chosen_set) }
493a0301c0dSLemover  }
494a0301c0dSLemover}
495