16d5ddbceSLemover/*************************************************************************************** 26d5ddbceSLemover* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3f320e0f0SYinan Xu* Copyright (c) 2020-2021 Peng Cheng Laboratory 46d5ddbceSLemover* 56d5ddbceSLemover* XiangShan is licensed under Mulan PSL v2. 66d5ddbceSLemover* You can use this software according to the terms and conditions of the Mulan PSL v2. 76d5ddbceSLemover* You may obtain a copy of Mulan PSL v2 at: 86d5ddbceSLemover* http://license.coscl.org.cn/MulanPSL2 9f1fe8698SLemover 106d5ddbceSLemover* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 116d5ddbceSLemover* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 126d5ddbceSLemover* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 136d5ddbceSLemover* 146d5ddbceSLemover* See the Mulan PSL v2 for more details. 156d5ddbceSLemover***************************************************************************************/ 166d5ddbceSLemover 176d5ddbceSLemoverpackage xiangshan.cache.mmu 186d5ddbceSLemover 198891a219SYinan Xuimport org.chipsalliance.cde.config.Parameters 206d5ddbceSLemoverimport chisel3._ 216d5ddbceSLemoverimport chisel3.util._ 225ab1b84dSHaoyuan Fengimport difftest._ 23a0301c0dSLemoverimport freechips.rocketchip.util.SRAMAnnotation 246d5ddbceSLemoverimport xiangshan._ 256d5ddbceSLemoverimport utils._ 263c02ee8fSwakafaimport utility._ 27f1fe8698SLemoverimport xiangshan.backend.fu.{PMPChecker, PMPReqBundle, PMPConfig => XSPMPConfig} 289aca92b9SYinan Xuimport xiangshan.backend.rob.RobPtr 296d5ddbceSLemoverimport xiangshan.backend.fu.util.HasCSRConst 30f1fe8698SLemoverimport freechips.rocketchip.rocket.PMPConfig 316d5ddbceSLemover 32f1fe8698SLemover/** TLB module 33f1fe8698SLemover * support block request and non-block request io at the same time 34f1fe8698SLemover * return paddr at next cycle, then go for pmp/pma check 35f1fe8698SLemover * @param Width: The number of requestors 36f1fe8698SLemover * @param Block: Blocked or not for each requestor ports 37f1fe8698SLemover * @param q: TLB Parameters, like entry number, each TLB has its own parameters 38f1fe8698SLemover * @param p: XiangShan Paramemters, like XLEN 39f1fe8698SLemover */ 40a0301c0dSLemover 4103efd994Shappy-lxclass TLB(Width: Int, nRespDups: Int = 1, Block: Seq[Boolean], q: TLBParameters)(implicit p: Parameters) extends TlbModule 42f1fe8698SLemover with HasCSRConst 43f1fe8698SLemover with HasPerfEvents 44f1fe8698SLemover{ 4503efd994Shappy-lx val io = IO(new TlbIO(Width, nRespDups, q)) 46a0301c0dSLemover 476d5ddbceSLemover val req = io.requestor.map(_.req) 486d5ddbceSLemover val resp = io.requestor.map(_.resp) 496d5ddbceSLemover val ptw = io.ptw 50b6982e83SLemover val pmp = io.pmp 518744445eSMaxpicca-Li val refill_to_mem = io.refill_to_mem 526d5ddbceSLemover 53f1fe8698SLemover /** Sfence.vma & Svinval 54f1fe8698SLemover * Sfence.vma will 1. flush old entries 2. flush inflight 3. flush pipe 55f1fe8698SLemover * Svinval will 1. flush old entries 2. flush inflight 56f1fe8698SLemover * So, Svinval will not flush pipe, which means 57f1fe8698SLemover * it should not drop reqs from pipe and should return right resp 58f1fe8698SLemover */ 59f1fe8698SLemover val sfence = DelayN(io.sfence, q.fenceDelay) 606d5ddbceSLemover val csr = io.csr 61f1fe8698SLemover val satp = DelayN(io.csr.satp, q.fenceDelay) 62d0de7e4aSpeixiaokun val vsatp = DelayN(io.csr.vsatp, q.fenceDelay) 63d0de7e4aSpeixiaokun val hgatp = DelayN(io.csr.hgatp, q.fenceDelay) 64d0de7e4aSpeixiaokun 65d0de7e4aSpeixiaokun val flush_mmu = DelayN(sfence.valid || csr.satp.changed || csr.vsatp.changed || csr.hgatp.changed, q.fenceDelay) 66f1fe8698SLemover val mmu_flush_pipe = DelayN(sfence.valid && sfence.bits.flushPipe, q.fenceDelay) // for svinval, won't flush pipe 67f1fe8698SLemover val flush_pipe = io.flushPipe 68f1fe8698SLemover 69ffa711ffSpeixiaokun val req_in = req 70ffa711ffSpeixiaokun val req_out = req.map(a => RegEnable(a.bits, a.fire())) 71ffa711ffSpeixiaokun val req_out_v = (0 until Width).map(i => ValidHold(req_in(i).fire && !req_in(i).bits.kill, resp(i).fire, flush_pipe(i))) 72ffa711ffSpeixiaokun 73ffa711ffSpeixiaokun val isHyperInst = (0 until Width).map(i => req_out_v(i) && req_out(i).hyperinst) 7450c7aa78Speixiaokun 75f1fe8698SLemover // ATTENTION: csr and flush from backend are delayed. csr should not be later than flush. 76f1fe8698SLemover // because, csr will influence tlb behavior. 77a0301c0dSLemover val ifecth = if (q.fetchi) true.B else false.B 78d0de7e4aSpeixiaokun val mode_tmp = if (q.useDmode) csr.priv.dmode else csr.priv.imode 79d0de7e4aSpeixiaokun val mode = (0 until Width).map(i => Mux(isHyperInst(i), csr.priv.spvp, mode_tmp)) 80d0de7e4aSpeixiaokun val virt = csr.priv.virt 81d0de7e4aSpeixiaokun val sum = (0 until Width).map(i => Mux(virt || isHyperInst(i), io.csr.priv.vsum, io.csr.priv.sum)) 82d0de7e4aSpeixiaokun val mxr = (0 until Width).map(i => Mux(virt || isHyperInst(i), io.csr.priv.vmxr || io.csr.priv.mxr, io.csr.priv.mxr)) 83ffa711ffSpeixiaokun val req_in_s2xlate = (0 until Width).map(i => MuxCase(noS2xlate, Seq( 84ffa711ffSpeixiaokun (!(virt || req_in(i).bits.hyperinst)) -> noS2xlate, 85ffa711ffSpeixiaokun (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage, 86ffa711ffSpeixiaokun (vsatp.mode === 0.U) -> onlyStage2, 87ffa711ffSpeixiaokun (hgatp.mode === 0.U) -> onlyStage1 88ffa711ffSpeixiaokun ))) 89ffa711ffSpeixiaokun val req_out_s2xlate = (0 until Width).map(i => MuxCase(noS2xlate, Seq( 903106de0aSpeixiaokun (!(virt || isHyperInst(i))) -> noS2xlate, 913106de0aSpeixiaokun (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage, 923106de0aSpeixiaokun (vsatp.mode === 0.U) -> onlyStage2, 933106de0aSpeixiaokun (hgatp.mode === 0.U) -> onlyStage1 943106de0aSpeixiaokun ))) 95*e9027bcdSpeixiaokun val need_gpa = RegInit(false.B) 96*e9027bcdSpeixiaokun val need_gpa_vpn = Reg(UInt(vpnLen.W)) 97*e9027bcdSpeixiaokun val need_gpa_gvpn = Reg(UInt(vpnLen.W)) 98*e9027bcdSpeixiaokun val hasGpf = Wire(Vec(Width, Bool())) 99d0de7e4aSpeixiaokun 1006d5ddbceSLemover // val vmEnable = satp.mode === 8.U // && (mode < ModeM) // FIXME: fix me when boot xv6/linux... 101d0de7e4aSpeixiaokun val vmEnable = (0 until Width).map(i => if (EnbaleTlbDebug) (satp.mode === 8.U) 102d0de7e4aSpeixiaokun else (satp.mode === 8.U) && (mode(i) < ModeM)) 103d0de7e4aSpeixiaokun val s2xlateEnable = (0 until Width).map(i => (isHyperInst(i) || virt) && (vsatp.mode === 8.U || hgatp.mode === 8.U) && (mode(i) < ModeM)) 104d0de7e4aSpeixiaokun val portTranslateEnable = (0 until Width).map(i => (vmEnable(i) || s2xlateEnable(i)) && RegNext(!req(i).bits.no_translate)) 1056d5ddbceSLemover 1066d5ddbceSLemover 107*e9027bcdSpeixiaokun val refill = (0 until Width).map(i => ptw.resp.fire && !(need_gpa && need_gpa_vpn === ptw.resp.bits.getVpn) && !flush_mmu && (vmEnable(i) || ptw.resp.bits.s2xlate =/= noS2xlate)) 108eb4bf3f2Speixiaokun refill_to_mem := DontCare 10903efd994Shappy-lx val entries = Module(new TlbStorageWrapper(Width, q, nRespDups)) 110f1fe8698SLemover entries.io.base_connect(sfence, csr, satp) 111f1fe8698SLemover if (q.outReplace) { io.replace <> entries.io.replace } 1126d5ddbceSLemover for (i <- 0 until Width) { 113ffa711ffSpeixiaokun entries.io.r_req_apply(io.requestor(i).req.valid, get_pn(req_in(i).bits.vaddr), i, req_in_s2xlate(i)) 114cca17e78Speixiaokun entries.io.w_apply(refill(i), ptw.resp.bits) 1158744445eSMaxpicca-Li resp(i).bits.debug.isFirstIssue := RegNext(req(i).bits.debug.isFirstIssue) 1168744445eSMaxpicca-Li resp(i).bits.debug.robIdx := RegNext(req(i).bits.debug.robIdx) 117a0301c0dSLemover } 118*e9027bcdSpeixiaokun 119f1fe8698SLemover // read TLB, get hit/miss, paddr, perm bits 120f1fe8698SLemover val readResult = (0 until Width).map(TLBRead(_)) 121f1fe8698SLemover val hitVec = readResult.map(_._1) 122f1fe8698SLemover val missVec = readResult.map(_._2) 123f1fe8698SLemover val pmp_addr = readResult.map(_._3) 124f9ac118cSHaoyuan Feng val perm = readResult.map(_._4) 1253106de0aSpeixiaokun val g_perm = readResult.map(_._5) 126f1fe8698SLemover // check pmp use paddr (for timing optization, use pmp_addr here) 127f1fe8698SLemover // check permisson 128f1fe8698SLemover (0 until Width).foreach{i => 129f1fe8698SLemover pmp_check(pmp_addr(i), req_out(i).size, req_out(i).cmd, i) 13003efd994Shappy-lx for (d <- 0 until nRespDups) { 131ffa711ffSpeixiaokun perm_check(perm(i)(d), req_out(i).cmd, i, d, g_perm(i)(d), req_out(i).hlvx, req_out_s2xlate(i)) 13203efd994Shappy-lx } 133c3d5cfb3Speixiaokun hasGpf(i) := resp(i).bits.excp(0).gpf.ld || resp(i).bits.excp(0).gpf.st || resp(i).bits.excp(0).gpf.instr 134f1fe8698SLemover } 1356d5ddbceSLemover 136f1fe8698SLemover // handle block or non-block io 137f1fe8698SLemover // for non-block io, just return the above result, send miss to ptw 138f1fe8698SLemover // for block io, hold the request, send miss to ptw, 139f1fe8698SLemover // when ptw back, return the result 140f1fe8698SLemover (0 until Width) foreach {i => 141f1fe8698SLemover if (Block(i)) handle_block(i) 142f1fe8698SLemover else handle_nonblock(i) 143f1fe8698SLemover } 144f1fe8698SLemover io.ptw.resp.ready := true.B 145a0301c0dSLemover 146f1fe8698SLemover /************************ main body above | method/log/perf below ****************************/ 147f1fe8698SLemover def TLBRead(i: Int) = { 148ffa711ffSpeixiaokun val (e_hit, e_ppn, e_perm, e_g_perm, e_s2xlate) = entries.io.r_resp_apply(i) 149ffa711ffSpeixiaokun val (p_hit, p_ppn, p_perm, p_gvpn, p_g_perm, p_s2xlate) = ptw_resp_bypass(get_pn(req_in(i).bits.vaddr), req_in_s2xlate(i)) 150292bea3fSWilliam Wang val enable = portTranslateEnable(i) 151f86480a7Speixiaokun val isOnlys2xlate = req_out_s2xlate(i) === onlyStage2 1523106de0aSpeixiaokun val resp_gpa_refill = RegInit(false.B) 1533106de0aSpeixiaokun val need_gpa_vpn_hit = RegEnable(need_gpa_vpn === get_pn(req_in(i).bits.vaddr), req_in(i).fire()) 154f86480a7Speixiaokun when (io.requestor(i).resp.valid && hasGpf(i) && need_gpa === false.B && !need_gpa_vpn_hit && !isOnlys2xlate) { 155c3d5cfb3Speixiaokun need_gpa := true.B 1563106de0aSpeixiaokun need_gpa_vpn := get_pn(req_out(i).vaddr) 1573106de0aSpeixiaokun resp_gpa_refill := false.B 158c3d5cfb3Speixiaokun } 1593106de0aSpeixiaokun when (ptw.resp.fire && need_gpa && need_gpa_vpn === ptw.resp.bits.getVpn) { 1603106de0aSpeixiaokun need_gpa_gvpn := Mux(ptw.resp.bits.s2xlate === onlyStage2, Cat(ptw.resp.bits.s1.entry.tag, ptw.resp.bits.s1.ppn_low(OHToUInt(ptw.resp.bits.s1.pteidx))), ptw.resp.bits.s2.entry.tag) 1613106de0aSpeixiaokun resp_gpa_refill := true.B 1623106de0aSpeixiaokun } 1633106de0aSpeixiaokun 164933ec998Speixiaokun when (hasGpf(i) && resp_gpa_refill && need_gpa_vpn_hit){ 165c3d5cfb3Speixiaokun need_gpa := false.B 166c3d5cfb3Speixiaokun } 167c3d5cfb3Speixiaokun 168cb8f2f2aSLemover val hit = e_hit || p_hit 169f86480a7Speixiaokun val miss = (!hit && enable) || hasGpf(i) && !(resp_gpa_refill && need_gpa_vpn_hit) && !isOnlys2xlate 170f1fe8698SLemover hit.suggestName(s"hit_read_${i}") 171f1fe8698SLemover miss.suggestName(s"miss_read_${i}") 1726d5ddbceSLemover 173f1fe8698SLemover val vaddr = SignExt(req_out(i).vaddr, PAddrBits) 174f1fe8698SLemover resp(i).bits.miss := miss 175935edac4STang Haojin resp(i).bits.ptwBack := ptw.resp.fire 1768744445eSMaxpicca-Li resp(i).bits.memidx := RegNext(req_in(i).bits.memidx) 1776d5ddbceSLemover 17803efd994Shappy-lx val ppn = WireInit(VecInit(Seq.fill(nRespDups)(0.U(ppnLen.W)))) 17903efd994Shappy-lx val perm = WireInit(VecInit(Seq.fill(nRespDups)(0.U.asTypeOf(new TlbPermBundle)))) 18082978df9Speixiaokun val gvpn = WireInit(VecInit(Seq.fill(nRespDups)(0.U(vpnLen.W)))) 181d0de7e4aSpeixiaokun val g_perm = WireInit(VecInit(Seq.fill(nRespDups)(0.U.asTypeOf(new TlbPermBundle)))) 18250c7aa78Speixiaokun val r_s2xlate = WireInit(VecInit(Seq.fill(nRespDups)(0.U(2.W)))) 18303efd994Shappy-lx for (d <- 0 until nRespDups) { 18403efd994Shappy-lx ppn(d) := Mux(p_hit, p_ppn, e_ppn(d)) 18503efd994Shappy-lx perm(d) := Mux(p_hit, p_perm, e_perm(d)) 186c794d992Speixiaokun gvpn(d) := Mux(hasGpf(i), need_gpa_gvpn, 0.U) 187d0de7e4aSpeixiaokun g_perm(d) := Mux(p_hit, p_g_perm, e_g_perm(d)) 18850c7aa78Speixiaokun r_s2xlate(d) := Mux(p_hit, p_s2xlate, e_s2xlate(d)) 18903efd994Shappy-lx val paddr = Cat(ppn(d), get_off(req_out(i).vaddr)) 190d0de7e4aSpeixiaokun val gpaddr = Cat(gvpn(d), get_off(req_out(i).vaddr)) 191292bea3fSWilliam Wang resp(i).bits.paddr(d) := Mux(enable, paddr, vaddr) 19250c7aa78Speixiaokun resp(i).bits.gpaddr(d) := Mux(r_s2xlate(d) === onlyStage2, vaddr, gpaddr) 19303efd994Shappy-lx } 19403efd994Shappy-lx 19503efd994Shappy-lx XSDebug(req_out_v(i), p"(${i.U}) hit:${hit} miss:${miss} ppn:${Hexadecimal(ppn(0))} perm:${perm(0)}\n") 19603efd994Shappy-lx 197f9ac118cSHaoyuan Feng val pmp_paddr = resp(i).bits.paddr(0) 198f1fe8698SLemover 1993106de0aSpeixiaokun (hit, miss, pmp_paddr, perm, g_perm) 200f1fe8698SLemover } 201f1fe8698SLemover 202f1fe8698SLemover def pmp_check(addr: UInt, size: UInt, cmd: UInt, idx: Int): Unit = { 203f1fe8698SLemover pmp(idx).valid := resp(idx).valid 204f1fe8698SLemover pmp(idx).bits.addr := addr 205f1fe8698SLemover pmp(idx).bits.size := size 206f1fe8698SLemover pmp(idx).bits.cmd := cmd 207f1fe8698SLemover } 208f1fe8698SLemover 209d0de7e4aSpeixiaokun def perm_check(perm: TlbPermBundle, cmd: UInt, idx: Int, nDups: Int, g_perm: TlbPermBundle, hlvx: Bool, s2xlate: UInt) = { 2105b7ef044SLemover // for timing optimization, pmp check is divided into dynamic and static 2115b7ef044SLemover // dynamic: superpage (or full-connected reg entries) -> check pmp when translation done 2125b7ef044SLemover // static: 4K pages (or sram entries) -> check pmp with pre-checked results 213c3d5cfb3Speixiaokun val hasS2xlate = s2xlate =/= noS2xlate 21407f77bf0Speixiaokun val onlyS2 = s2xlate === onlyStage2 215d0de7e4aSpeixiaokun val af = perm.af || (hasS2xlate && g_perm.af) 216d0de7e4aSpeixiaokun 217d0de7e4aSpeixiaokun // Stage 1 perm check 218382a2ebdSpeixiaokun val pf = perm.pf || (hlvx && !perm.x) 219f1fe8698SLemover val ldUpdate = !perm.a && TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd) // update A/D through exception 220f1fe8698SLemover val stUpdate = (!perm.a || !perm.d) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd)) // update A/D through exception 221f1fe8698SLemover val instrUpdate = !perm.a && TlbCmd.isExec(cmd) // update A/D through exception 222d0de7e4aSpeixiaokun val modeCheck = !(mode(idx) === ModeU && !perm.u || mode(idx) === ModeS && perm.u && (!sum(idx) || ifecth)) 223d0de7e4aSpeixiaokun val ldPermFail = !(modeCheck && (perm.r || mxr(idx) && perm.x)) 224a79fef67Swakafa val stPermFail = !(modeCheck && perm.w) 225a79fef67Swakafa val instrPermFail = !(modeCheck && perm.x) 226f1fe8698SLemover val ldPf = (ldPermFail || pf) && (TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd)) 227f1fe8698SLemover val stPf = (stPermFail || pf) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd)) 228f1fe8698SLemover val instrPf = (instrPermFail || pf) && TlbCmd.isExec(cmd) 229d0de7e4aSpeixiaokun val s1_valid = portTranslateEnable(idx) && !onlyS2 230d0de7e4aSpeixiaokun 231d0de7e4aSpeixiaokun // Stage 2 perm check 232382a2ebdSpeixiaokun val gpf = g_perm.pf || (hlvx && !g_perm.x) 233d0de7e4aSpeixiaokun val g_ldUpdate = !g_perm.a && TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd) 234d0de7e4aSpeixiaokun val g_stUpdate = (!g_perm.a || !g_perm.d) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd)) 235d0de7e4aSpeixiaokun val g_instrUpdate = !g_perm.a && TlbCmd.isExec(cmd) 236d0de7e4aSpeixiaokun val g_ldPermFail = !(g_perm.r || io.csr.priv.mxr && g_perm.x) 237d0de7e4aSpeixiaokun val g_stPermFail = !g_perm.w 238d0de7e4aSpeixiaokun val g_instrPermFail = !g_perm.x 239d0de7e4aSpeixiaokun val ldGpf = (g_ldPermFail || gpf) && (TlbCmd.isRead(cmd) && !TlbCmd.isAmo(cmd)) 240d0de7e4aSpeixiaokun val stGpf = (g_stPermFail || gpf) && (TlbCmd.isWrite(cmd) || TlbCmd.isAmo(cmd)) 241d0de7e4aSpeixiaokun val instrGpf = (g_instrPermFail || gpf) && TlbCmd.isExec(cmd) 24244f8e3e4Speixiaokun val s2_valid = hasS2xlate && portTranslateEnable(idx) 243d0de7e4aSpeixiaokun 244d0de7e4aSpeixiaokun val fault_valid = s1_valid || s2_valid 245d0de7e4aSpeixiaokun 246c794d992Speixiaokun // when pf and gpf can't happens simultaneously 247c794d992Speixiaokun val hasPf = (ldPf || ldUpdate || stPf || stUpdate || instrPf || instrUpdate) && s1_valid && !af 248d0de7e4aSpeixiaokun resp(idx).bits.excp(nDups).pf.ld := (ldPf || ldUpdate) && s1_valid && !af 249d0de7e4aSpeixiaokun resp(idx).bits.excp(nDups).pf.st := (stPf || stUpdate) && s1_valid && !af 250d0de7e4aSpeixiaokun resp(idx).bits.excp(nDups).pf.instr := (instrPf || instrUpdate) && s1_valid && !af 251b6982e83SLemover // NOTE: pf need && with !af, page fault has higher priority than access fault 252b6982e83SLemover // but ptw may also have access fault, then af happens, the translation is wrong. 253b6982e83SLemover // In this case, pf has lower priority than af 2546d5ddbceSLemover 255c794d992Speixiaokun resp(idx).bits.excp(nDups).gpf.ld := (ldGpf || g_ldUpdate) && s2_valid && !af && !hasPf 256c794d992Speixiaokun resp(idx).bits.excp(nDups).gpf.st := (stGpf || g_stUpdate) && s2_valid && !af && !hasPf 257c794d992Speixiaokun resp(idx).bits.excp(nDups).gpf.instr := (instrGpf || g_instrUpdate) && s2_valid && !af && !hasPf 258d0de7e4aSpeixiaokun 259f9ac118cSHaoyuan Feng resp(idx).bits.excp(nDups).af.ld := af && TlbCmd.isRead(cmd) && fault_valid 260f9ac118cSHaoyuan Feng resp(idx).bits.excp(nDups).af.st := af && TlbCmd.isWrite(cmd) && fault_valid 261f9ac118cSHaoyuan Feng resp(idx).bits.excp(nDups).af.instr := af && TlbCmd.isExec(cmd) && fault_valid 262d0de7e4aSpeixiaokun 263d0de7e4aSpeixiaokun 2646d5ddbceSLemover } 2656d5ddbceSLemover 266f1fe8698SLemover def handle_nonblock(idx: Int): Unit = { 267f1fe8698SLemover io.requestor(idx).resp.valid := req_out_v(idx) 268f1fe8698SLemover io.requestor(idx).req.ready := io.requestor(idx).resp.ready // should always be true 2699930e66fSLemover XSError(!io.requestor(idx).resp.ready, s"${q.name} port ${idx} is non-block, resp.ready must be true.B") 270cb8f2f2aSLemover 271c3d5cfb3Speixiaokun val req_need_gpa = hasGpf(idx) 272d0de7e4aSpeixiaokun val req_s2xlate = Wire(UInt(2.W)) 27382978df9Speixiaokun req_s2xlate := MuxCase(noS2xlate, Seq( 274496c751cSpeixiaokun (!(virt || req_out(idx).hyperinst)) -> noS2xlate, 275cca17e78Speixiaokun (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage, 276cca17e78Speixiaokun (vsatp.mode === 0.U) -> onlyStage2, 277c3d5cfb3Speixiaokun (hgatp.mode === 0.U || req_need_gpa) -> onlyStage1 27882978df9Speixiaokun )) 279933ec998Speixiaokun val ptw_s2xlate = ptw.resp.bits.s2xlate 280933ec998Speixiaokun val has_s2xlate = ptw_s2xlate =/= noS2xlate 28182978df9Speixiaokun val onlyS2 = ptw_s2xlate === onlyStage2 282933ec998Speixiaokun val ptw_s1_hit = ptw.resp.bits.s1.hit(get_pn(req_out(idx).vaddr), Mux(has_s2xlate, io.csr.vsatp.asid, io.csr.satp.asid), io.csr.hgatp.asid, true, false, has_s2xlate) 283d0de7e4aSpeixiaokun val ptw_s2_hit = ptw.resp.bits.s2.hit(get_pn(req_out(idx).vaddr), io.csr.hgatp.asid) 284d0de7e4aSpeixiaokun val ptw_just_back = ptw.resp.fire && req_s2xlate === ptw_s2xlate && Mux(onlyS2, ptw_s2_hit, ptw_s1_hit) 285185e6164SHaoyuan Feng val ptw_already_back = RegNext(ptw.resp.fire) && RegNext(ptw.resp.bits).hit(get_pn(req_out(idx).vaddr), asid = io.csr.satp.asid, allType = true) 286c3d5cfb3Speixiaokun io.ptw.req(idx).valid := req_out_v(idx) && (missVec(idx)) && !(ptw_just_back || ptw_already_back) // TODO: remove the regnext, timing 287185e6164SHaoyuan Feng io.tlbreplay(idx) := req_out_v(idx) && missVec(idx) && (ptw_just_back || ptw_already_back) 288185e6164SHaoyuan Feng when (io.requestor(idx).req_kill && RegNext(io.requestor(idx).req.fire)) { 289c3b763d0SYinan Xu io.ptw.req(idx).valid := false.B 290185e6164SHaoyuan Feng io.tlbreplay(idx) := true.B 291c3b763d0SYinan Xu } 292185e6164SHaoyuan Feng io.ptw.req(idx).bits.vpn := get_pn(req_out(idx).vaddr) 293d0de7e4aSpeixiaokun io.ptw.req(idx).bits.s2xlate := RegNext(req_s2xlate) 294185e6164SHaoyuan Feng io.ptw.req(idx).bits.memidx := req_out(idx).memidx 295149086eaSLemover } 296a0301c0dSLemover 297f1fe8698SLemover def handle_block(idx: Int): Unit = { 298f1fe8698SLemover // three valid: 1.if exist a entry; 2.if sent to ptw; 3.unset resp.valid 299935edac4STang Haojin io.requestor(idx).req.ready := !req_out_v(idx) || io.requestor(idx).resp.fire 300f1fe8698SLemover // req_out_v for if there is a request, may long latency, fixme 301f1fe8698SLemover 302f1fe8698SLemover // miss request entries 303c3d5cfb3Speixiaokun val req_need_gpa = hasGpf(idx) 304f1fe8698SLemover val miss_req_vpn = get_pn(req_out(idx).vaddr) 3058744445eSMaxpicca-Li val miss_req_memidx = req_out(idx).memidx 306d0de7e4aSpeixiaokun val miss_req_s2xlate = Wire(UInt(2.W)) 30782978df9Speixiaokun miss_req_s2xlate := MuxCase(noS2xlate, Seq( 308496c751cSpeixiaokun (!(virt || req_out(idx).hyperinst)) -> noS2xlate, 309cca17e78Speixiaokun (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage, 310cca17e78Speixiaokun (vsatp.mode === 0.U) -> onlyStage2, 311c3d5cfb3Speixiaokun (hgatp.mode === 0.U || req_need_gpa) -> onlyStage1 31282978df9Speixiaokun )) 313c3d5cfb3Speixiaokun val miss_req_s2xlate_reg = RegEnable(miss_req_s2xlate, io.ptw.req(idx).fire()) 314c3d5cfb3Speixiaokun val hasS2xlate = miss_req_s2xlate_reg =/= noS2xlate 315c3d5cfb3Speixiaokun val onlyS2 = miss_req_s2xlate_reg === onlyStage2 31682978df9Speixiaokun val hit_s1 = io.ptw.resp.bits.s1.hit(miss_req_vpn, Mux(hasS2xlate, io.csr.vsatp.asid, io.csr.satp.asid), io.csr.hgatp.asid, allType = true, false, hasS2xlate) 317d0de7e4aSpeixiaokun val hit_s2 = io.ptw.resp.bits.s2.hit(miss_req_vpn, io.csr.hgatp.asid) 318c3d5cfb3Speixiaokun val hit = Mux(onlyS2, hit_s2, hit_s1) && io.ptw.resp.valid && miss_req_s2xlate_reg === io.ptw.resp.bits.s2xlate 319f1fe8698SLemover 320f1fe8698SLemover val new_coming = RegNext(req_in(idx).fire && !req_in(idx).bits.kill && !flush_pipe(idx), false.B) 321f1fe8698SLemover val miss_wire = new_coming && missVec(idx) 322935edac4STang Haojin val miss_v = ValidHoldBypass(miss_wire, resp(idx).fire, flush_pipe(idx)) 323f1fe8698SLemover val miss_req_v = ValidHoldBypass(miss_wire || (miss_v && flush_mmu && !mmu_flush_pipe), 324935edac4STang Haojin io.ptw.req(idx).fire || resp(idx).fire, flush_pipe(idx)) 325f1fe8698SLemover 326f1fe8698SLemover // when ptw resp, check if hit, reset miss_v, resp to lsu/ifu 327292bea3fSWilliam Wang resp(idx).valid := req_out_v(idx) && !(miss_v && portTranslateEnable(idx)) 328935edac4STang Haojin when (io.ptw.resp.fire && hit && req_out_v(idx) && portTranslateEnable(idx)) { 329d0de7e4aSpeixiaokun val stage1 = io.ptw.resp.bits.s1 330d0de7e4aSpeixiaokun val stage2 = io.ptw.resp.bits.s2 331d0de7e4aSpeixiaokun val s2xlate = io.ptw.resp.bits.s2xlate 332f1fe8698SLemover resp(idx).valid := true.B 333c3d5cfb3Speixiaokun resp(idx).bits.miss := false.B 334d0de7e4aSpeixiaokun val s1_paddr = Cat(stage1.genPPN(get_pn(req_out(idx).vaddr)), get_off(req_out(idx).vaddr)) 3357e664aa3Speixiaokun val s2_paddr = Cat(stage2.genPPNS2(get_pn(req_out(idx).vaddr)), get_off(req_out(idx).vaddr)) 33603efd994Shappy-lx for (d <- 0 until nRespDups) { 33782978df9Speixiaokun resp(idx).bits.paddr(d) := Mux(s2xlate =/= noS2xlate, s2_paddr, s1_paddr) 338d0de7e4aSpeixiaokun resp(idx).bits.gpaddr(d) := s1_paddr 339cca17e78Speixiaokun perm_check(stage1, req_out(idx).cmd, idx, d, stage2, req_out(idx).hlvx, s2xlate) 34003efd994Shappy-lx } 34103efd994Shappy-lx pmp_check(resp(idx).bits.paddr(0), req_out(idx).size, req_out(idx).cmd, idx) 342f1fe8698SLemover 343f1fe8698SLemover // NOTE: the unfiltered req would be handled by Repeater 344f1fe8698SLemover } 345f1fe8698SLemover assert(RegNext(!resp(idx).valid || resp(idx).ready, true.B), "when tlb resp valid, ready should be true, must") 346f1fe8698SLemover assert(RegNext(req_out_v(idx) || !(miss_v || miss_req_v), true.B), "when not req_out_v, should not set miss_v/miss_req_v") 347f1fe8698SLemover 348f1fe8698SLemover val ptw_req = io.ptw.req(idx) 349f1fe8698SLemover ptw_req.valid := miss_req_v 350f1fe8698SLemover ptw_req.bits.vpn := miss_req_vpn 351d0de7e4aSpeixiaokun ptw_req.bits.s2xlate := miss_req_s2xlate 3528744445eSMaxpicca-Li ptw_req.bits.memidx := miss_req_memidx 353f1fe8698SLemover 354185e6164SHaoyuan Feng io.tlbreplay(idx) := false.B 355185e6164SHaoyuan Feng 356f1fe8698SLemover // NOTE: when flush pipe, tlb should abandon last req 357f1fe8698SLemover // however, some outside modules like icache, dont care flushPipe, and still waiting for tlb resp 358f1fe8698SLemover // just resp valid and raise page fault to go through. The pipe(ifu) will abandon it. 359f1fe8698SLemover if (!q.outsideRecvFlush) { 360292bea3fSWilliam Wang when (req_out_v(idx) && flush_pipe(idx) && portTranslateEnable(idx)) { 361f1fe8698SLemover resp(idx).valid := true.B 36203efd994Shappy-lx for (d <- 0 until nRespDups) { 36303efd994Shappy-lx resp(idx).bits.excp(d).pf.ld := true.B // sfence happened, pf for not to use this addr 36403efd994Shappy-lx resp(idx).bits.excp(d).pf.st := true.B 36503efd994Shappy-lx resp(idx).bits.excp(d).pf.instr := true.B 36603efd994Shappy-lx } 367f1fe8698SLemover } 368f1fe8698SLemover } 369f1fe8698SLemover } 370cb8f2f2aSLemover 371cb8f2f2aSLemover // when ptw resp, tlb at refill_idx maybe set to miss by force. 372cb8f2f2aSLemover // Bypass ptw resp to check. 373d0de7e4aSpeixiaokun def ptw_resp_bypass(vpn: UInt, s2xlate: UInt) = { 374cca17e78Speixiaokun val hasS2xlate = s2xlate =/= noS2xlate 375cca17e78Speixiaokun val onlyS2 = s2xlate === onlyStage2 376c3d5cfb3Speixiaokun val onlyS1 = s2xlate === onlyStage1 377d0de7e4aSpeixiaokun val s2xlate_hit = s2xlate === ptw.resp.bits.s2xlate 378d0de7e4aSpeixiaokun val normal_hit = ptw.resp.bits.s1.hit(vpn, Mux(hasS2xlate, io.csr.vsatp.asid, io.csr.satp.asid), io.csr.hgatp.asid, true, false, hasS2xlate) 379d0de7e4aSpeixiaokun val onlyS2_hit = ptw.resp.bits.s2.hit(vpn, io.csr.hgatp.asid) 380d0de7e4aSpeixiaokun val p_hit = RegNext(Mux(onlyS2, onlyS2_hit, normal_hit) && io.ptw.resp.fire && s2xlate_hit) 381d0de7e4aSpeixiaokun val ppn_s1 = ptw.resp.bits.s1.genPPN(vpn) 3827e664aa3Speixiaokun val ppn_s2 = ptw.resp.bits.s2.genPPNS2(vpn) 383d0de7e4aSpeixiaokun val p_ppn = RegEnable(Mux(hasS2xlate, ppn_s2, ppn_s1), io.ptw.resp.fire) 384d0de7e4aSpeixiaokun val p_perm = RegEnable(ptwresp_to_tlbperm(ptw.resp.bits.s1), io.ptw.resp.fire) 385c3d5cfb3Speixiaokun val p_gvpn = RegEnable(Mux(onlyS1, Cat(ptw.resp.bits.s1.entry.tag, ptw.resp.bits.s1.ppn_low(OHToUInt(ptw.resp.bits.s1.pteidx))), ptw.resp.bits.s2.entry.tag), io.ptw.resp.fire) 386d0de7e4aSpeixiaokun val p_g_perm = RegEnable(hptwresp_to_tlbperm(ptw.resp.bits.s2), io.ptw.resp.fire) 387d0de7e4aSpeixiaokun val p_s2xlate = RegEnable(ptw.resp.bits.s2xlate, io.ptw.resp.fire) 388d0de7e4aSpeixiaokun (p_hit, p_ppn, p_perm, p_gvpn, p_g_perm, p_s2xlate) 389cb8f2f2aSLemover } 390cb8f2f2aSLemover 391f1fe8698SLemover // assert 392f1fe8698SLemover for(i <- 0 until Width) { 393f1fe8698SLemover TimeOutAssert(req_out_v(i) && !resp(i).valid, timeOutThreshold, s"{q.name} port{i} long time no resp valid.") 394149086eaSLemover } 395a0301c0dSLemover 396f1fe8698SLemover // perf event 397935edac4STang Haojin val result_ok = req_in.map(a => RegNext(a.fire)) 398f1fe8698SLemover val perfEvents = 399f1fe8698SLemover Seq( 400935edac4STang Haojin ("access", PopCount((0 until Width).map{i => if (Block(i)) io.requestor(i).req.fire else portTranslateEnable(i) && result_ok(i) })), 401935edac4STang Haojin ("miss ", PopCount((0 until Width).map{i => if (Block(i)) portTranslateEnable(i) && result_ok(i) && missVec(i) else ptw.req(i).fire })), 402a0301c0dSLemover ) 403f1fe8698SLemover generatePerfEvent() 404a0301c0dSLemover 405f1fe8698SLemover // perf log 4066d5ddbceSLemover for (i <- 0 until Width) { 407f1fe8698SLemover if (Block(i)) { 408292bea3fSWilliam Wang XSPerfAccumulate(s"access${i}",result_ok(i) && portTranslateEnable(i)) 409f1fe8698SLemover XSPerfAccumulate(s"miss${i}", result_ok(i) && missVec(i)) 4106d5ddbceSLemover } else { 411292bea3fSWilliam Wang XSPerfAccumulate("first_access" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && RegNext(req(i).bits.debug.isFirstIssue)) 412292bea3fSWilliam Wang XSPerfAccumulate("access" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i)) 413292bea3fSWilliam Wang XSPerfAccumulate("first_miss" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && missVec(i) && RegNext(req(i).bits.debug.isFirstIssue)) 414292bea3fSWilliam Wang XSPerfAccumulate("miss" + Integer.toString(i, 10), result_ok(i) && portTranslateEnable(i) && missVec(i)) 415a0301c0dSLemover } 4166d5ddbceSLemover } 417935edac4STang Haojin XSPerfAccumulate("ptw_resp_count", ptw.resp.fire) 418cca17e78Speixiaokun XSPerfAccumulate("ptw_resp_pf_count", ptw.resp.fire && ptw.resp.bits.s1.pf) 4196d5ddbceSLemover 4206d5ddbceSLemover // Log 4216d5ddbceSLemover for(i <- 0 until Width) { 4226d5ddbceSLemover XSDebug(req(i).valid, p"req(${i.U}): (${req(i).valid} ${req(i).ready}) ${req(i).bits}\n") 4236d5ddbceSLemover XSDebug(resp(i).valid, p"resp(${i.U}): (${resp(i).valid} ${resp(i).ready}) ${resp(i).bits}\n") 4246d5ddbceSLemover } 4256d5ddbceSLemover 426f1fe8698SLemover XSDebug(io.sfence.valid, p"Sfence: ${io.sfence}\n") 427f1fe8698SLemover XSDebug(ParallelOR(req_out_v) || ptw.resp.valid, p"vmEnable:${vmEnable} hit:${Binary(VecInit(hitVec).asUInt)} miss:${Binary(VecInit(missVec).asUInt)}\n") 4286d5ddbceSLemover for (i <- ptw.req.indices) { 429935edac4STang Haojin XSDebug(ptw.req(i).fire, p"L2TLB req:${ptw.req(i).bits}\n") 4306d5ddbceSLemover } 43192e3bfefSLemover XSDebug(ptw.resp.valid, p"L2TLB resp:${ptw.resp.bits} (v:${ptw.resp.valid}r:${ptw.resp.ready}) \n") 4326d5ddbceSLemover 433f9ac118cSHaoyuan Feng println(s"${q.name}: page: ${q.NWays} ${q.Associative} ${q.Replacer.get}") 434a0301c0dSLemover 4355ab1b84dSHaoyuan Feng if (env.EnableDifftest) { 4365ab1b84dSHaoyuan Feng for (i <- 0 until Width) { 4375ab1b84dSHaoyuan Feng val pf = io.requestor(i).resp.bits.excp(0).pf.instr || io.requestor(i).resp.bits.excp(0).pf.st || io.requestor(i).resp.bits.excp(0).pf.ld 438d0de7e4aSpeixiaokun val gpf = io.requestor(i).resp.bits.excp(0).gpf.instr || io.requestor(i).resp.bits.excp(0).gpf.st || io.requestor(i).resp.bits.excp(0).gpf.ld 4395ab1b84dSHaoyuan Feng val af = io.requestor(i).resp.bits.excp(0).af.instr || io.requestor(i).resp.bits.excp(0).af.st || io.requestor(i).resp.bits.excp(0).af.ld 4407d45a146SYinan Xu val difftest = DifftestModule(new DiffL1TLBEvent) 441254e4960SHaoyuan Feng difftest.coreid := io.hartId 442d0de7e4aSpeixiaokun difftest.valid := RegNext(io.requestor(i).req.fire) && !io.requestor(i).req_kill && io.requestor(i).resp.fire && !io.requestor(i).resp.bits.miss && !pf && !af && !gpf && portTranslateEnable(i) 4437d45a146SYinan Xu if (!Seq("itlb", "ldtlb", "sttlb").contains(q.name)) { 4447d45a146SYinan Xu difftest.valid := false.B 4457d45a146SYinan Xu } 4467d45a146SYinan Xu difftest.index := TLBDiffId(p(XSCoreParamsKey).HartId).U 447d0de7e4aSpeixiaokun difftest.satp := io.csr.satp 4487d45a146SYinan Xu difftest.vpn := RegNext(get_pn(req_in(i).bits.vaddr)) 4497d45a146SYinan Xu difftest.ppn := get_pn(io.requestor(i).resp.bits.paddr(0)) 45087d0ba30Speixiaokun difftest.satp := Cat(io.csr.satp.mode, io.csr.satp.asid, io.csr.satp.ppn) 45187d0ba30Speixiaokun difftest.vsatp := Cat(io.csr.vsatp.mode, io.csr.vsatp.asid, io.csr.vsatp.ppn) 45287d0ba30Speixiaokun difftest.hgatp := Cat(io.csr.hgatp.mode, io.csr.hgatp.asid, io.csr.hgatp.ppn) 453dd103903Speixiaokun val req_need_gpa = gpf 454dd103903Speixiaokun val req_s2xlate = Wire(UInt(2.W)) 455dd103903Speixiaokun req_s2xlate := MuxCase(noS2xlate, Seq( 456dd103903Speixiaokun (!(virt || RegNext(req_in(i).bits.hyperinst))) -> noS2xlate, 457cca17e78Speixiaokun (vsatp.mode =/= 0.U && hgatp.mode =/= 0.U) -> allStage, 458cca17e78Speixiaokun (vsatp.mode === 0.U) -> onlyStage2, 459dd103903Speixiaokun (hgatp.mode === 0.U || req_need_gpa) -> onlyStage1 46082978df9Speixiaokun )) 461dd103903Speixiaokun difftest.s2xlate := req_s2xlate 4627d45a146SYinan Xu } 4635ab1b84dSHaoyuan Feng } 4645ab1b84dSHaoyuan Feng} 4655ab1b84dSHaoyuan Feng 4667d45a146SYinan Xuobject TLBDiffId { 4677d45a146SYinan Xu var i: Int = 0 4687d45a146SYinan Xu var lastHartId: Int = -1 4697d45a146SYinan Xu def apply(hartId: Int): Int = { 4707d45a146SYinan Xu if (lastHartId != hartId) { 4717d45a146SYinan Xu i = 0 4727d45a146SYinan Xu lastHartId = hartId 4737d45a146SYinan Xu } 4747d45a146SYinan Xu i += 1 4757d45a146SYinan Xu i - 1 4767d45a146SYinan Xu } 477f1fe8698SLemover} 4781ca0e4f3SYinan Xu 47903efd994Shappy-lxclass TLBNonBlock(Width: Int, nRespDups: Int = 1, q: TLBParameters)(implicit p: Parameters) extends TLB(Width, nRespDups, Seq.fill(Width)(false), q) 48003efd994Shappy-lxclass TLBBLock(Width: Int, nRespDups: Int = 1, q: TLBParameters)(implicit p: Parameters) extends TLB(Width, nRespDups, Seq.fill(Width)(true), q) 4816d5ddbceSLemover 482a0301c0dSLemoverclass TlbReplace(Width: Int, q: TLBParameters)(implicit p: Parameters) extends TlbModule { 483a0301c0dSLemover val io = IO(new TlbReplaceIO(Width, q)) 484a0301c0dSLemover 485f9ac118cSHaoyuan Feng if (q.Associative == "fa") { 486f9ac118cSHaoyuan Feng val re = ReplacementPolicy.fromString(q.Replacer, q.NWays) 487f9ac118cSHaoyuan Feng re.access(io.page.access.map(_.touch_ways)) 488f9ac118cSHaoyuan Feng io.page.refillIdx := re.way 489a0301c0dSLemover } else { // set-acco && plru 490f9ac118cSHaoyuan Feng val re = ReplacementPolicy.fromString(q.Replacer, q.NSets, q.NWays) 491f9ac118cSHaoyuan Feng re.access(io.page.access.map(_.sets), io.page.access.map(_.touch_ways)) 492f9ac118cSHaoyuan Feng io.page.refillIdx := { if (q.NWays == 1) 0.U else re.way(io.page.chosen_set) } 493a0301c0dSLemover } 494a0301c0dSLemover} 495