16d5ddbceSLemover/*************************************************************************************** 26d5ddbceSLemover* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3f320e0f0SYinan Xu* Copyright (c) 2020-2021 Peng Cheng Laboratory 46d5ddbceSLemover* 56d5ddbceSLemover* XiangShan is licensed under Mulan PSL v2. 66d5ddbceSLemover* You can use this software according to the terms and conditions of the Mulan PSL v2. 76d5ddbceSLemover* You may obtain a copy of Mulan PSL v2 at: 86d5ddbceSLemover* http://license.coscl.org.cn/MulanPSL2 96d5ddbceSLemover* 106d5ddbceSLemover* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 116d5ddbceSLemover* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 126d5ddbceSLemover* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 136d5ddbceSLemover* 146d5ddbceSLemover* See the Mulan PSL v2 for more details. 156d5ddbceSLemover***************************************************************************************/ 166d5ddbceSLemover 176d5ddbceSLemoverpackage xiangshan.cache.mmu 186d5ddbceSLemover 198891a219SYinan Xuimport org.chipsalliance.cde.config.Parameters 206d5ddbceSLemoverimport chisel3._ 216d5ddbceSLemoverimport chisel3.util._ 226d5ddbceSLemoverimport xiangshan._ 236d5ddbceSLemoverimport xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 246d5ddbceSLemoverimport utils._ 253c02ee8fSwakafaimport utility._ 266d5ddbceSLemoverimport freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 276d5ddbceSLemoverimport freechips.rocketchip.tilelink._ 28b6982e83SLemoverimport xiangshan.backend.fu.{PMPReqBundle, PMPRespBundle} 296d5ddbceSLemover 3092e3bfefSLemover/** Page Table Walk is divided into two parts 3192e3bfefSLemover * One, PTW: page walk for pde, except for leaf entries, one by one 3292e3bfefSLemover * Two, LLPTW: page walk for pte, only the leaf entries(4KB), in parallel 336d5ddbceSLemover */ 3492e3bfefSLemover 3592e3bfefSLemover 3692e3bfefSLemover/** PTW : page table walker 3792e3bfefSLemover * a finite state machine 3892e3bfefSLemover * only take 1GB and 2MB page walks 3992e3bfefSLemover * or in other words, except the last level(leaf) 4092e3bfefSLemover **/ 4192e3bfefSLemoverclass PTWIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 426d5ddbceSLemover val req = Flipped(DecoupledIO(new Bundle { 4345f497a4Shappy-lx val req_info = new L2TlbInnerBundle() 446d5ddbceSLemover val l1Hit = Bool() 454c0e0181SXiaokun-Pei val ppn = UInt(gvpnLen.W) 4630104977Speixiaokun val stage1Hit = Bool() 4730104977Speixiaokun val stage1 = new PtwMergeResp 486d5ddbceSLemover })) 496d5ddbceSLemover val resp = DecoupledIO(new Bundle { 50bc063562SLemover val source = UInt(bSourceWidth.W) 51eb4bf3f2Speixiaokun val s2xlate = UInt(2.W) 5263632028SHaoyuan Feng val resp = new PtwMergeResp 53d0de7e4aSpeixiaokun val h_resp = new HptwResp 546d5ddbceSLemover }) 556d5ddbceSLemover 5692e3bfefSLemover val llptw = DecoupledIO(new LLPTWInBundle()) 579c503409SLemover // NOTE: llptw change from "connect to llptw" to "connect to page cache" 589c503409SLemover // to avoid corner case that caused duplicate entries 59cc5a5f22SLemover 60d0de7e4aSpeixiaokun val hptw = new Bundle { 61d0de7e4aSpeixiaokun val req = DecoupledIO(new Bundle { 62eb4bf3f2Speixiaokun val source = UInt(bSourceWidth.W) 63d0de7e4aSpeixiaokun val id = UInt(log2Up(l2tlbParams.llptwsize).W) 6482978df9Speixiaokun val gvpn = UInt(vpnLen.W) 65d0de7e4aSpeixiaokun }) 66d0de7e4aSpeixiaokun val resp = Flipped(Valid(new Bundle { 67d0de7e4aSpeixiaokun val h_resp = Output(new HptwResp) 68d0de7e4aSpeixiaokun })) 69d0de7e4aSpeixiaokun } 706d5ddbceSLemover val mem = new Bundle { 71b848eea5SLemover val req = DecoupledIO(new L2TlbMemReqBundle()) 725854c1edSLemover val resp = Flipped(ValidIO(UInt(XLEN.W))) 73cc5a5f22SLemover val mask = Input(Bool()) 746d5ddbceSLemover } 75b6982e83SLemover val pmp = new Bundle { 76b6982e83SLemover val req = ValidIO(new PMPReqBundle()) 77b6982e83SLemover val resp = Flipped(new PMPRespBundle()) 78b6982e83SLemover } 796d5ddbceSLemover 806d5ddbceSLemover val refill = Output(new Bundle { 8145f497a4Shappy-lx val req_info = new L2TlbInnerBundle() 826d5ddbceSLemover val level = UInt(log2Up(Level).W) 836d5ddbceSLemover }) 846d5ddbceSLemover} 856d5ddbceSLemover 8692e3bfefSLemoverclass PTW()(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 8792e3bfefSLemover val io = IO(new PTWIO) 886d5ddbceSLemover val sfence = io.sfence 896d5ddbceSLemover val mem = io.mem 90d0de7e4aSpeixiaokun val req_s2xlate = Reg(UInt(2.W)) 9103c1129fSpeixiaokun val enableS2xlate = req_s2xlate =/= noS2xlate 9203c1129fSpeixiaokun val onlyS1xlate = req_s2xlate === onlyStage1 9303c1129fSpeixiaokun val onlyS2xlate = req_s2xlate === onlyStage2 94d0de7e4aSpeixiaokun 95d0de7e4aSpeixiaokun val satp = Mux(enableS2xlate, io.csr.vsatp, io.csr.satp) 96d0de7e4aSpeixiaokun val hgatp = io.csr.hgatp 975c5f442fSXiaokun-Pei val flush = io.sfence.valid || io.csr.satp.changed || io.csr.vsatp.changed || io.csr.hgatp.changed 98d0de7e4aSpeixiaokun val s2xlate = enableS2xlate && !onlyS1xlate 996d5ddbceSLemover val level = RegInit(0.U(log2Up(Level).W)) 100b6982e83SLemover val af_level = RegInit(0.U(log2Up(Level).W)) // access fault return this level 1014c0e0181SXiaokun-Pei val ppn = Reg(UInt(gvpnLen.W)) 1024c0e0181SXiaokun-Pei val vpn = Reg(UInt(vpnLen.W)) // vpn or gvpn(onlyS2xlate) 1036d5ddbceSLemover val levelNext = level + 1.U 1046d5ddbceSLemover val l1Hit = Reg(Bool()) 105*505c893aSXiaokun-Pei val pte_valid = RegInit(false.B) // avoid the x states 106*505c893aSXiaokun-Pei val fake_pte = 0.U.asTypeOf(new PteBundle()) 107*505c893aSXiaokun-Pei fake_pte.perm.v := true.B 108*505c893aSXiaokun-Pei fake_pte.perm.r := true.B 109*505c893aSXiaokun-Pei fake_pte.perm.w := true.B 110*505c893aSXiaokun-Pei fake_pte.perm.x := true.B 111*505c893aSXiaokun-Pei val pte = Mux(pte_valid, mem.resp.bits.asTypeOf(new PteBundle()), fake_pte) 11244b79566SXiaokun-Pei // s/w register 11344b79566SXiaokun-Pei val s_pmp_check = RegInit(true.B) 11444b79566SXiaokun-Pei val s_mem_req = RegInit(true.B) 11544b79566SXiaokun-Pei val s_llptw_req = RegInit(true.B) 11644b79566SXiaokun-Pei val w_mem_resp = RegInit(true.B) 117d0de7e4aSpeixiaokun val s_hptw_req = RegInit(true.B) 118d0de7e4aSpeixiaokun val w_hptw_resp = RegInit(true.B) 119d0de7e4aSpeixiaokun val s_last_hptw_req = RegInit(true.B) 120d0de7e4aSpeixiaokun val w_last_hptw_resp = RegInit(true.B) 12144b79566SXiaokun-Pei // for updating "level" 12244b79566SXiaokun-Pei val mem_addr_update = RegInit(false.B) 12344b79566SXiaokun-Pei 12444b79566SXiaokun-Pei val idle = RegInit(true.B) 1252a906a65SHaoyuan Feng val finish = WireInit(false.B) 1262a906a65SHaoyuan Feng val sent_to_pmp = idle === false.B && (s_pmp_check === false.B || mem_addr_update) && !finish 12744b79566SXiaokun-Pei 128d0de7e4aSpeixiaokun val pageFault = pte.isPf(level) 12944b79566SXiaokun-Pei val accessFault = RegEnable(io.pmp.resp.ld || io.pmp.resp.mmio, sent_to_pmp) 1306d5ddbceSLemover 131d0de7e4aSpeixiaokun val hptw_pageFault = RegInit(false.B) 132d0de7e4aSpeixiaokun val hptw_accessFault = RegInit(false.B) 133d0de7e4aSpeixiaokun val last_s2xlate = RegInit(false.B) 1343222d00fSpeixiaokun val stage1Hit = RegEnable(io.req.bits.stage1Hit, io.req.fire) 1353222d00fSpeixiaokun val stage1 = RegEnable(io.req.bits.stage1, io.req.fire) 13609280d15Speixiaokun val hptw_resp_stage2 = Reg(Bool()) 137d0de7e4aSpeixiaokun 1384c0e0181SXiaokun-Pei val ppn_af = Mux(s2xlate, pte.isStage1Af(), pte.isAf()) // In two-stage address translation, stage 1 ppn is a vpn for host, so don't need to check ppn_high 1397263b595SXiaokun-Pei val guest_fault = hptw_pageFault || hptw_accessFault 1407263b595SXiaokun-Pei val find_pte = pte.isLeaf() || ppn_af || pageFault 14144b79566SXiaokun-Pei val to_find_pte = level === 1.U && find_pte === false.B 142935edac4STang Haojin val source = RegEnable(io.req.bits.req_info.source, io.req.fire) 1436d5ddbceSLemover 1446d5ddbceSLemover val l1addr = MakeAddr(satp.ppn, getVpnn(vpn, 2)) 1454c0e0181SXiaokun-Pei val l2addr = MakeAddr(Mux(l1Hit, ppn, pte.getPPN()), getVpnn(vpn, 1)) 146b6982e83SLemover val mem_addr = Mux(af_level === 0.U, l1addr, l2addr) 14744b79566SXiaokun-Pei 1483222d00fSpeixiaokun val hptw_resp = RegEnable(io.hptw.resp.bits.h_resp, io.hptw.resp.fire) 149c0991f6aSpeixiaokun val gpaddr = MuxCase(mem_addr, Seq( 150c0991f6aSpeixiaokun stage1Hit -> Cat(stage1.genPPN(), 0.U(offLen.W)), 151c0991f6aSpeixiaokun onlyS2xlate -> Cat(vpn, 0.U(offLen.W)), 1524c0e0181SXiaokun-Pei !s_last_hptw_req -> Cat(MuxLookup(level, pte.getPPN())(Seq( 1534c0e0181SXiaokun-Pei 0.U -> Cat(pte.getPPN()(gvpnLen - 1, vpnnLen * 2), vpn(vpnnLen * 2 - 1, 0)), 1544c0e0181SXiaokun-Pei 1.U -> Cat(pte.getPPN()(gvpnLen - 1, vpnnLen), vpn(vpnnLen - 1, 0) 155dcb10e8fSBL-GS ))), 156dcb10e8fSBL-GS 0.U(offLen.W)) 157c0991f6aSpeixiaokun )) 158cda84113Speixiaokun val hpaddr = Cat(hptw_resp.genPPNS2(get_pn(gpaddr)), get_off(gpaddr)) 159d0de7e4aSpeixiaokun 16044b79566SXiaokun-Pei io.req.ready := idle 16130104977Speixiaokun val ptw_resp = Wire(new PtwMergeResp) 162*505c893aSXiaokun-Pei ptw_resp.apply(pageFault && !accessFault && !ppn_af, accessFault || ppn_af, Mux(accessFault, af_level,level), pte, vpn, satp.asid, hgatp.asid, vpn(sectortlbwidth - 1, 0), not_super = false) 16344b79566SXiaokun-Pei 1640dfe2fbdSpeixiaokun val normal_resp = idle === false.B && mem_addr_update && !last_s2xlate && (guest_fault || (w_mem_resp && find_pte) || (s_pmp_check && accessFault) || onlyS2xlate ) 16509280d15Speixiaokun val stageHit_resp = idle === false.B && hptw_resp_stage2 16609280d15Speixiaokun io.resp.valid := Mux(stage1Hit, stageHit_resp, normal_resp) 16744b79566SXiaokun-Pei io.resp.bits.source := source 16830104977Speixiaokun io.resp.bits.resp := Mux(stage1Hit, stage1, ptw_resp) 16979d4b70cSpeixiaokun io.resp.bits.h_resp := hptw_resp 1706315ba2aSpeixiaokun io.resp.bits.s2xlate := req_s2xlate 17144b79566SXiaokun-Pei 17244b79566SXiaokun-Pei io.llptw.valid := s_llptw_req === false.B && to_find_pte && !accessFault 17344b79566SXiaokun-Pei io.llptw.bits.req_info.source := source 17444b79566SXiaokun-Pei io.llptw.bits.req_info.vpn := vpn 17582978df9Speixiaokun io.llptw.bits.req_info.s2xlate := req_s2xlate 176eb4bf3f2Speixiaokun io.llptw.bits.ppn := DontCare 17744b79566SXiaokun-Pei 178b6982e83SLemover io.pmp.req.valid := DontCare // samecycle, do not use valid 179d0de7e4aSpeixiaokun io.pmp.req.bits.addr := Mux(s2xlate, hpaddr, mem_addr) 180b6982e83SLemover io.pmp.req.bits.size := 3.U // TODO: fix it 181b6982e83SLemover io.pmp.req.bits.cmd := TlbCmd.read 182b6982e83SLemover 18344b79566SXiaokun-Pei mem.req.valid := s_mem_req === false.B && !mem.mask && !accessFault && s_pmp_check 184d0de7e4aSpeixiaokun mem.req.bits.addr := Mux(s2xlate, hpaddr, mem_addr) 185bc063562SLemover mem.req.bits.id := FsmReqID.U(bMemID.W) 18683d93d53Speixiaokun mem.req.bits.hptw_bypassed := false.B 1876d5ddbceSLemover 1884ed5afbdSXiaokun-Pei io.refill.req_info.s2xlate := req_s2xlate 18945f497a4Shappy-lx io.refill.req_info.vpn := vpn 1906d5ddbceSLemover io.refill.level := level 19145f497a4Shappy-lx io.refill.req_info.source := source 1926d5ddbceSLemover 193d0de7e4aSpeixiaokun io.hptw.req.valid := !s_hptw_req || !s_last_hptw_req 194d0de7e4aSpeixiaokun io.hptw.req.bits.id := FsmReqID.U(bMemID.W) 195dcb10e8fSBL-GS io.hptw.req.bits.gvpn := get_pn(gpaddr) 196eb4bf3f2Speixiaokun io.hptw.req.bits.source := source 197d0de7e4aSpeixiaokun 1983222d00fSpeixiaokun when (io.req.fire && io.req.bits.stage1Hit){ 19930104977Speixiaokun idle := false.B 20061c5d636Speixiaokun req_s2xlate := io.req.bits.req_info.s2xlate 20130104977Speixiaokun s_hptw_req := false.B 20209280d15Speixiaokun hptw_resp_stage2 := false.B 2036bb8be21SXiaokun-Pei last_s2xlate := false.B 2040dfe2fbdSpeixiaokun hptw_pageFault := false.B 2050dfe2fbdSpeixiaokun hptw_accessFault := false.B 20630104977Speixiaokun } 207d0de7e4aSpeixiaokun 2083222d00fSpeixiaokun when (io.hptw.resp.fire && w_hptw_resp === false.B && stage1Hit){ 20930104977Speixiaokun w_hptw_resp := true.B 21009280d15Speixiaokun hptw_resp_stage2 := true.B 21130104977Speixiaokun } 21230104977Speixiaokun 2133222d00fSpeixiaokun when (io.resp.fire && stage1Hit){ 21430104977Speixiaokun idle := true.B 21530104977Speixiaokun } 21630104977Speixiaokun 2173222d00fSpeixiaokun when (io.req.fire && !io.req.bits.stage1Hit){ 21844b79566SXiaokun-Pei val req = io.req.bits 21944b79566SXiaokun-Pei level := Mux(req.l1Hit, 1.U, 0.U) 22044b79566SXiaokun-Pei af_level := Mux(req.l1Hit, 1.U, 0.U) 22144b79566SXiaokun-Pei ppn := Mux(req.l1Hit, io.req.bits.ppn, satp.ppn) 22244b79566SXiaokun-Pei vpn := io.req.bits.req_info.vpn 22344b79566SXiaokun-Pei l1Hit := req.l1Hit 22444b79566SXiaokun-Pei accessFault := false.B 22544b79566SXiaokun-Pei idle := false.B 226d0de7e4aSpeixiaokun hptw_pageFault := false.B 2277263b595SXiaokun-Pei hptw_accessFault := false.B 228cc72e3f5SXiaokun-Pei pte_valid := false.B 22950c7aa78Speixiaokun req_s2xlate := io.req.bits.req_info.s2xlate 23082978df9Speixiaokun when(io.req.bits.req_info.s2xlate =/= noS2xlate && io.req.bits.req_info.s2xlate =/= onlyStage1){ 231d0de7e4aSpeixiaokun last_s2xlate := true.B 232d0de7e4aSpeixiaokun s_hptw_req := false.B 233d0de7e4aSpeixiaokun }.otherwise { 2346bb8be21SXiaokun-Pei last_s2xlate := false.B 235d0de7e4aSpeixiaokun s_pmp_check := false.B 236d0de7e4aSpeixiaokun } 237d0de7e4aSpeixiaokun } 238d0de7e4aSpeixiaokun 2393222d00fSpeixiaokun when(io.hptw.req.fire && s_hptw_req === false.B){ 240d0de7e4aSpeixiaokun s_hptw_req := true.B 241d0de7e4aSpeixiaokun w_hptw_resp := false.B 242d0de7e4aSpeixiaokun } 243d0de7e4aSpeixiaokun 2443222d00fSpeixiaokun when(io.hptw.resp.fire && w_hptw_resp === false.B && !stage1Hit) { 245d0de7e4aSpeixiaokun hptw_pageFault := io.hptw.resp.bits.h_resp.gpf 246d0de7e4aSpeixiaokun hptw_accessFault := io.hptw.resp.bits.h_resp.gaf 247d0de7e4aSpeixiaokun w_hptw_resp := true.B 2483b805a93SXiaokun-Pei when(onlyS2xlate){ 249d0de7e4aSpeixiaokun mem_addr_update := true.B 250d0de7e4aSpeixiaokun last_s2xlate := false.B 2513b805a93SXiaokun-Pei }.elsewhen(!(io.hptw.resp.bits.h_resp.gpf || io.hptw.resp.bits.h_resp.gaf)) { 252d0de7e4aSpeixiaokun s_pmp_check := false.B 253d0de7e4aSpeixiaokun } 254d0de7e4aSpeixiaokun } 255d0de7e4aSpeixiaokun 2563222d00fSpeixiaokun when(io.hptw.req.fire && s_last_hptw_req === false.B) { 257d0de7e4aSpeixiaokun w_last_hptw_resp := false.B 258d0de7e4aSpeixiaokun s_last_hptw_req := true.B 259d0de7e4aSpeixiaokun } 260d0de7e4aSpeixiaokun 2613222d00fSpeixiaokun when(io.hptw.resp.fire && w_last_hptw_resp === false.B){ 262d0de7e4aSpeixiaokun hptw_pageFault := io.hptw.resp.bits.h_resp.gpf 263d0de7e4aSpeixiaokun hptw_accessFault := io.hptw.resp.bits.h_resp.gaf 264d0de7e4aSpeixiaokun w_last_hptw_resp := true.B 265d0de7e4aSpeixiaokun mem_addr_update := true.B 266d0de7e4aSpeixiaokun last_s2xlate := false.B 26744b79566SXiaokun-Pei } 26844b79566SXiaokun-Pei 26944b79566SXiaokun-Pei when(sent_to_pmp && mem_addr_update === false.B){ 27044b79566SXiaokun-Pei s_mem_req := false.B 27144b79566SXiaokun-Pei s_pmp_check := true.B 27244b79566SXiaokun-Pei } 27344b79566SXiaokun-Pei 27444b79566SXiaokun-Pei when(accessFault && idle === false.B){ 27544b79566SXiaokun-Pei s_pmp_check := true.B 27644b79566SXiaokun-Pei s_mem_req := true.B 27744b79566SXiaokun-Pei w_mem_resp := true.B 27844b79566SXiaokun-Pei s_llptw_req := true.B 279d0de7e4aSpeixiaokun s_hptw_req := true.B 280d0de7e4aSpeixiaokun w_hptw_resp := true.B 281d0de7e4aSpeixiaokun s_last_hptw_req := true.B 282d0de7e4aSpeixiaokun w_last_hptw_resp := true.B 28344b79566SXiaokun-Pei mem_addr_update := true.B 284d0de7e4aSpeixiaokun last_s2xlate := false.B 28544b79566SXiaokun-Pei } 28644b79566SXiaokun-Pei 2877263b595SXiaokun-Pei when(guest_fault && idle === false.B){ 2887263b595SXiaokun-Pei s_pmp_check := true.B 2897263b595SXiaokun-Pei s_mem_req := true.B 2907263b595SXiaokun-Pei w_mem_resp := true.B 2917263b595SXiaokun-Pei s_llptw_req := true.B 2927263b595SXiaokun-Pei s_hptw_req := true.B 2937263b595SXiaokun-Pei w_hptw_resp := true.B 2947263b595SXiaokun-Pei s_last_hptw_req := true.B 2957263b595SXiaokun-Pei w_last_hptw_resp := true.B 2967263b595SXiaokun-Pei mem_addr_update := true.B 2977263b595SXiaokun-Pei last_s2xlate := false.B 2987263b595SXiaokun-Pei } 2997263b595SXiaokun-Pei 300935edac4STang Haojin when (mem.req.fire){ 30144b79566SXiaokun-Pei s_mem_req := true.B 30244b79566SXiaokun-Pei w_mem_resp := false.B 30344b79566SXiaokun-Pei } 30444b79566SXiaokun-Pei 305935edac4STang Haojin when(mem.resp.fire && w_mem_resp === false.B){ 30644b79566SXiaokun-Pei w_mem_resp := true.B 30744b79566SXiaokun-Pei af_level := af_level + 1.U 30844b79566SXiaokun-Pei s_llptw_req := false.B 30944b79566SXiaokun-Pei mem_addr_update := true.B 310cc72e3f5SXiaokun-Pei pte_valid := true.B 31144b79566SXiaokun-Pei } 31244b79566SXiaokun-Pei 31344b79566SXiaokun-Pei when(mem_addr_update){ 3140dfe2fbdSpeixiaokun when(level === 0.U && !onlyS2xlate && !(guest_fault || find_pte || accessFault)){ 31544b79566SXiaokun-Pei level := levelNext 316d0de7e4aSpeixiaokun when(s2xlate){ 317d0de7e4aSpeixiaokun s_hptw_req := false.B 318d0de7e4aSpeixiaokun }.otherwise{ 31944b79566SXiaokun-Pei s_mem_req := false.B 320d0de7e4aSpeixiaokun } 32144b79566SXiaokun-Pei s_llptw_req := true.B 32244b79566SXiaokun-Pei mem_addr_update := false.B 3232a906a65SHaoyuan Feng }.elsewhen(io.llptw.valid){ 324935edac4STang Haojin when(io.llptw.fire) { 32544b79566SXiaokun-Pei idle := true.B 32644b79566SXiaokun-Pei s_llptw_req := true.B 32744b79566SXiaokun-Pei mem_addr_update := false.B 328d0de7e4aSpeixiaokun last_s2xlate := false.B 3292a906a65SHaoyuan Feng } 3302a906a65SHaoyuan Feng finish := true.B 331d0de7e4aSpeixiaokun }.elsewhen(s2xlate && last_s2xlate === true.B) { 3327c26eb06SXiaokun-Pei when(accessFault || pageFault || ppn_af){ 3337c26eb06SXiaokun-Pei last_s2xlate := false.B 3347c26eb06SXiaokun-Pei }.otherwise{ 335d0de7e4aSpeixiaokun s_last_hptw_req := false.B 336d0de7e4aSpeixiaokun mem_addr_update := false.B 3377c26eb06SXiaokun-Pei } 3382a906a65SHaoyuan Feng }.elsewhen(io.resp.valid){ 339935edac4STang Haojin when(io.resp.fire) { 34044b79566SXiaokun-Pei idle := true.B 34144b79566SXiaokun-Pei s_llptw_req := true.B 34244b79566SXiaokun-Pei mem_addr_update := false.B 34344b79566SXiaokun-Pei accessFault := false.B 34444b79566SXiaokun-Pei } 3452a906a65SHaoyuan Feng finish := true.B 3462a906a65SHaoyuan Feng } 34744b79566SXiaokun-Pei } 34844b79566SXiaokun-Pei 34944b79566SXiaokun-Pei 3505e237ba8SXiaokun-Pei when (flush) { 35144b79566SXiaokun-Pei idle := true.B 35244b79566SXiaokun-Pei s_pmp_check := true.B 35344b79566SXiaokun-Pei s_mem_req := true.B 35444b79566SXiaokun-Pei s_llptw_req := true.B 35544b79566SXiaokun-Pei w_mem_resp := true.B 35644b79566SXiaokun-Pei accessFault := false.B 357d826bce1SHaoyuan Feng mem_addr_update := false.B 358d0de7e4aSpeixiaokun s_hptw_req := true.B 359d0de7e4aSpeixiaokun w_hptw_resp := true.B 360d0de7e4aSpeixiaokun s_last_hptw_req := true.B 361d0de7e4aSpeixiaokun w_last_hptw_resp := true.B 36244b79566SXiaokun-Pei } 36344b79566SXiaokun-Pei 36444b79566SXiaokun-Pei 36544b79566SXiaokun-Pei XSDebug(p"[ptw] level:${level} notFound:${pageFault}\n") 3666d5ddbceSLemover 3676d5ddbceSLemover // perf 368935edac4STang Haojin XSPerfAccumulate("fsm_count", io.req.fire) 3696d5ddbceSLemover for (i <- 0 until PtwWidth) { 370935edac4STang Haojin XSPerfAccumulate(s"fsm_count_source${i}", io.req.fire && io.req.bits.req_info.source === i.U) 3716d5ddbceSLemover } 37244b79566SXiaokun-Pei XSPerfAccumulate("fsm_busy", !idle) 37344b79566SXiaokun-Pei XSPerfAccumulate("fsm_idle", idle) 3746d5ddbceSLemover XSPerfAccumulate("resp_blocked", io.resp.valid && !io.resp.ready) 375dd7fe201SHaoyuan Feng XSPerfAccumulate("ptw_ppn_af", io.resp.fire && ppn_af) 376935edac4STang Haojin XSPerfAccumulate("mem_count", mem.req.fire) 377935edac4STang Haojin XSPerfAccumulate("mem_cycle", BoolStopWatch(mem.req.fire, mem.resp.fire, true)) 3786d5ddbceSLemover XSPerfAccumulate("mem_blocked", mem.req.valid && !mem.req.ready) 379cc5a5f22SLemover 38044b79566SXiaokun-Pei TimeOutAssert(!idle, timeOutThreshold, "page table walker time out") 381cd365d4cSrvcoresjw 382cd365d4cSrvcoresjw val perfEvents = Seq( 383935edac4STang Haojin ("fsm_count ", io.req.fire ), 38444b79566SXiaokun-Pei ("fsm_busy ", !idle ), 38544b79566SXiaokun-Pei ("fsm_idle ", idle ), 386cd365d4cSrvcoresjw ("resp_blocked ", io.resp.valid && !io.resp.ready ), 387935edac4STang Haojin ("mem_count ", mem.req.fire ), 388935edac4STang Haojin ("mem_cycle ", BoolStopWatch(mem.req.fire, mem.resp.fire, true)), 389cd365d4cSrvcoresjw ("mem_blocked ", mem.req.valid && !mem.req.ready ), 390cd365d4cSrvcoresjw ) 3911ca0e4f3SYinan Xu generatePerfEvent() 3926d5ddbceSLemover} 39392e3bfefSLemover 39492e3bfefSLemover/*========================= LLPTW ==============================*/ 39592e3bfefSLemover 39692e3bfefSLemover/** LLPTW : Last Level Page Table Walker 39792e3bfefSLemover * the page walker that only takes 4KB(last level) page walk. 39892e3bfefSLemover **/ 39992e3bfefSLemover 40092e3bfefSLemoverclass LLPTWInBundle(implicit p: Parameters) extends XSBundle with HasPtwConst { 40192e3bfefSLemover val req_info = Output(new L2TlbInnerBundle()) 4024c0e0181SXiaokun-Pei val ppn = Output(UInt(gvpnLen.W)) 40392e3bfefSLemover} 40492e3bfefSLemover 40592e3bfefSLemoverclass LLPTWIO(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 40692e3bfefSLemover val in = Flipped(DecoupledIO(new LLPTWInBundle())) 40792e3bfefSLemover val out = DecoupledIO(new Bundle { 40892e3bfefSLemover val req_info = Output(new L2TlbInnerBundle()) 40992e3bfefSLemover val id = Output(UInt(bMemID.W)) 410d0de7e4aSpeixiaokun val h_resp = Output(new HptwResp) 4116979864eSXiaokun-Pei val first_s2xlate_fault = Output(Bool()) // Whether the first stage 2 translation occurs pf/af 41292e3bfefSLemover val af = Output(Bool()) 41392e3bfefSLemover }) 41492e3bfefSLemover val mem = new Bundle { 41592e3bfefSLemover val req = DecoupledIO(new L2TlbMemReqBundle()) 41692e3bfefSLemover val resp = Flipped(Valid(new Bundle { 41792e3bfefSLemover val id = Output(UInt(log2Up(l2tlbParams.llptwsize).W)) 418ce5f4200SGuanghui Hu val value = Output(UInt(blockBits.W)) 41992e3bfefSLemover })) 42092e3bfefSLemover val enq_ptr = Output(UInt(log2Ceil(l2tlbParams.llptwsize).W)) 42192e3bfefSLemover val buffer_it = Output(Vec(l2tlbParams.llptwsize, Bool())) 42292e3bfefSLemover val refill = Output(new L2TlbInnerBundle()) 42392e3bfefSLemover val req_mask = Input(Vec(l2tlbParams.llptwsize, Bool())) 42492e3bfefSLemover } 4257797f035SbugGenerator val cache = DecoupledIO(new L2TlbInnerBundle()) 42692e3bfefSLemover val pmp = new Bundle { 42792e3bfefSLemover val req = Valid(new PMPReqBundle()) 42892e3bfefSLemover val resp = Flipped(new PMPRespBundle()) 42992e3bfefSLemover } 430d0de7e4aSpeixiaokun val hptw = new Bundle { 431d0de7e4aSpeixiaokun val req = DecoupledIO(new Bundle{ 432eb4bf3f2Speixiaokun val source = UInt(bSourceWidth.W) 433d0de7e4aSpeixiaokun val id = UInt(log2Up(l2tlbParams.llptwsize).W) 43482978df9Speixiaokun val gvpn = UInt(vpnLen.W) 435d0de7e4aSpeixiaokun }) 436d0de7e4aSpeixiaokun val resp = Flipped(Valid(new Bundle { 437d0de7e4aSpeixiaokun val id = Output(UInt(log2Up(l2tlbParams.llptwsize).W)) 438d0de7e4aSpeixiaokun val h_resp = Output(new HptwResp) 439d0de7e4aSpeixiaokun })) 440d0de7e4aSpeixiaokun } 44192e3bfefSLemover} 44292e3bfefSLemover 44392e3bfefSLemoverclass LLPTWEntry(implicit p: Parameters) extends XSBundle with HasPtwConst { 44492e3bfefSLemover val req_info = new L2TlbInnerBundle() 4454c0e0181SXiaokun-Pei val ppn = UInt(gvpnLen.W) 44692e3bfefSLemover val wait_id = UInt(log2Up(l2tlbParams.llptwsize).W) 44792e3bfefSLemover val af = Bool() 448dc05c713Speixiaokun val hptw_resp = new HptwResp() 4496979864eSXiaokun-Pei val first_s2xlate_fault = Output(Bool()) 45092e3bfefSLemover} 45192e3bfefSLemover 45292e3bfefSLemover 45392e3bfefSLemoverclass LLPTW(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 45492e3bfefSLemover val io = IO(new LLPTWIO()) 45582978df9Speixiaokun val enableS2xlate = io.in.bits.req_info.s2xlate =/= noS2xlate 456d0de7e4aSpeixiaokun val satp = Mux(enableS2xlate, io.csr.vsatp, io.csr.satp) 45792e3bfefSLemover 4585c5f442fSXiaokun-Pei val flush = io.sfence.valid || io.csr.satp.changed || io.csr.vsatp.changed || io.csr.hgatp.changed 45992e3bfefSLemover val entries = Reg(Vec(l2tlbParams.llptwsize, new LLPTWEntry())) 460d0de7e4aSpeixiaokun val state_idle :: state_hptw_req :: state_hptw_resp :: state_addr_check :: state_mem_req :: state_mem_waiting :: state_mem_out :: state_last_hptw_req :: state_last_hptw_resp :: state_cache :: Nil = Enum(10) 46192e3bfefSLemover val state = RegInit(VecInit(Seq.fill(l2tlbParams.llptwsize)(state_idle))) 4627797f035SbugGenerator 46392e3bfefSLemover val is_emptys = state.map(_ === state_idle) 46492e3bfefSLemover val is_mems = state.map(_ === state_mem_req) 46592e3bfefSLemover val is_waiting = state.map(_ === state_mem_waiting) 46692e3bfefSLemover val is_having = state.map(_ === state_mem_out) 4677797f035SbugGenerator val is_cache = state.map(_ === state_cache) 468d0de7e4aSpeixiaokun val is_hptw_req = state.map(_ === state_hptw_req) 469d0de7e4aSpeixiaokun val is_last_hptw_req = state.map(_ === state_last_hptw_req) 470b7bdb307Speixiaokun val is_hptw_resp = state.map(_ === state_hptw_resp) 471b7bdb307Speixiaokun val is_last_hptw_resp = state.map(_ === state_last_hptw_resp) 47292e3bfefSLemover 473935edac4STang Haojin val full = !ParallelOR(is_emptys).asBool 47492e3bfefSLemover val enq_ptr = ParallelPriorityEncoder(is_emptys) 47592e3bfefSLemover 4767797f035SbugGenerator val mem_ptr = ParallelPriorityEncoder(is_having) // TODO: optimize timing, bad: entries -> ptr -> entry 4777be7e781Speixiaokun val mem_arb = Module(new RRArbiterInit(new LLPTWEntry(), l2tlbParams.llptwsize)) 47892e3bfefSLemover for (i <- 0 until l2tlbParams.llptwsize) { 47992e3bfefSLemover mem_arb.io.in(i).bits := entries(i) 48092e3bfefSLemover mem_arb.io.in(i).valid := is_mems(i) && !io.mem.req_mask(i) 48192e3bfefSLemover } 4822a1f48e7Speixiaokun 4832a1f48e7Speixiaokun // process hptw requests in serial 4847be7e781Speixiaokun val hyper_arb1 = Module(new RRArbiterInit(new LLPTWEntry(), l2tlbParams.llptwsize)) 485d0de7e4aSpeixiaokun for (i <- 0 until l2tlbParams.llptwsize) { 486d0de7e4aSpeixiaokun hyper_arb1.io.in(i).bits := entries(i) 4872a1f48e7Speixiaokun hyper_arb1.io.in(i).valid := is_hptw_req(i) && !(Cat(is_hptw_resp).orR) && !(Cat(is_last_hptw_resp).orR) 488d0de7e4aSpeixiaokun } 4897be7e781Speixiaokun val hyper_arb2 = Module(new RRArbiterInit(new LLPTWEntry(), l2tlbParams.llptwsize)) 490d0de7e4aSpeixiaokun for(i <- 0 until l2tlbParams.llptwsize) { 491d0de7e4aSpeixiaokun hyper_arb2.io.in(i).bits := entries(i) 4922a1f48e7Speixiaokun hyper_arb2.io.in(i).valid := is_last_hptw_req(i) && !(Cat(is_hptw_resp).orR) && !(Cat(is_last_hptw_resp).orR) 493d0de7e4aSpeixiaokun } 49492e3bfefSLemover 495f3034303SHaoyuan Feng val cache_ptr = ParallelMux(is_cache, (0 until l2tlbParams.llptwsize).map(_.U(log2Up(l2tlbParams.llptwsize).W))) 4967797f035SbugGenerator 49792e3bfefSLemover // duplicate req 49892e3bfefSLemover // to_wait: wait for the last to access mem, set to mem_resp 49992e3bfefSLemover // to_cache: the last is back just right now, set to mem_cache 50092e3bfefSLemover val dup_vec = state.indices.map(i => 501cca17e78Speixiaokun dup(io.in.bits.req_info.vpn, entries(i).req_info.vpn) && io.in.bits.req_info.s2xlate === entries(i).req_info.s2xlate 50292e3bfefSLemover ) 503cca17e78Speixiaokun val dup_req_fire = mem_arb.io.out.fire && dup(io.in.bits.req_info.vpn, mem_arb.io.out.bits.req_info.vpn) && io.in.bits.req_info.s2xlate === mem_arb.io.out.bits.req_info.s2xlate // dup with the req fire entry 5046979864eSXiaokun-Pei val dup_vec_wait = dup_vec.zip(is_waiting).map{case (d, w) => d && w} // dup with "mem_waiting" entries, sending mem req already 50592e3bfefSLemover val dup_vec_having = dup_vec.zipWithIndex.map{case (d, i) => d && is_having(i)} // dup with the "mem_out" entry recv the data just now 506951f37e5Speixiaokun val dup_vec_last_hptw = dup_vec.zipWithIndex.map{case (d, i) => d && (is_last_hptw_req(i) || is_last_hptw_resp(i))} 50792e3bfefSLemover val wait_id = Mux(dup_req_fire, mem_arb.io.chosen, ParallelMux(dup_vec_wait zip entries.map(_.wait_id))) 508935edac4STang Haojin val dup_wait_resp = io.mem.resp.fire && VecInit(dup_vec_wait)(io.mem.resp.bits.id) // dup with the entry that data coming next cycle 50992e3bfefSLemover val to_wait = Cat(dup_vec_wait).orR || dup_req_fire 510c6655c9aSXiaokun-Pei val to_mem_out = dup_wait_resp && ((entries(io.mem.resp.bits.id).req_info.s2xlate === noS2xlate) || (entries(io.mem.resp.bits.id).req_info.s2xlate === onlyStage1)) 511951f37e5Speixiaokun val to_cache = Cat(dup_vec_having).orR || Cat(dup_vec_last_hptw).orR 5126b742a19SXiaokun-Pei val to_hptw_req = io.in.bits.req_info.s2xlate === allStage 5136b742a19SXiaokun-Pei val to_last_hptw_req = dup_wait_resp && entries(io.mem.resp.bits.id).req_info.s2xlate === allStage 5149467c5f4Speixiaokun val last_hptw_req_id = io.mem.resp.bits.id 5154c0e0181SXiaokun-Pei val req_paddr = MakeAddr(io.in.bits.ppn(ppnLen-1, 0), getVpnn(io.in.bits.req_info.vpn, 0)) 5169467c5f4Speixiaokun val req_hpaddr = MakeAddr(entries(last_hptw_req_id).hptw_resp.genPPNS2(get_pn(req_paddr)), getVpnn(io.in.bits.req_info.vpn, 0)) 5179467c5f4Speixiaokun val index = Mux(entries(last_hptw_req_id).req_info.s2xlate === allStage, req_hpaddr, req_paddr)(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 5184c0e0181SXiaokun-Pei val last_hptw_req_ppn = io.mem.resp.bits.value.asTypeOf(Vec(blockBits / XLEN, new PteBundle()))(index).getPPN() 5197797f035SbugGenerator XSError(RegNext(dup_req_fire && Cat(dup_vec_wait).orR, init = false.B), "mem req but some entries already waiting, should not happed") 52092e3bfefSLemover 521935edac4STang Haojin XSError(io.in.fire && ((to_mem_out && to_cache) || (to_wait && to_cache)), "llptw enq, to cache conflict with to mem") 52292e3bfefSLemover val mem_resp_hit = RegInit(VecInit(Seq.fill(l2tlbParams.llptwsize)(false.B))) 5237274ec5cSpeixiaokun val enq_state_normal = MuxCase(state_addr_check, Seq( 5247274ec5cSpeixiaokun to_mem_out -> state_mem_out, // same to the blew, but the mem resp now 525871d1438Speixiaokun to_last_hptw_req -> state_last_hptw_req, 5267274ec5cSpeixiaokun to_wait -> state_mem_waiting, 5277274ec5cSpeixiaokun to_cache -> state_cache, 528871d1438Speixiaokun to_hptw_req -> state_hptw_req 5297274ec5cSpeixiaokun )) 5307797f035SbugGenerator val enq_state = Mux(from_pre(io.in.bits.req_info.source) && enq_state_normal =/= state_addr_check, state_idle, enq_state_normal) 531935edac4STang Haojin when (io.in.fire) { 53292e3bfefSLemover // if prefetch req does not need mem access, just give it up. 53392e3bfefSLemover // so there will be at most 1 + FilterSize entries that needs re-access page cache 53492e3bfefSLemover // so 2 + FilterSize is enough to avoid dead-lock 5357797f035SbugGenerator state(enq_ptr) := enq_state 53692e3bfefSLemover entries(enq_ptr).req_info := io.in.bits.req_info 5379467c5f4Speixiaokun entries(enq_ptr).ppn := Mux(to_last_hptw_req, last_hptw_req_ppn, io.in.bits.ppn) 53892e3bfefSLemover entries(enq_ptr).wait_id := Mux(to_wait, wait_id, enq_ptr) 53992e3bfefSLemover entries(enq_ptr).af := false.B 5402a1f48e7Speixiaokun entries(enq_ptr).hptw_resp := Mux(to_last_hptw_req, entries(last_hptw_req_id).hptw_resp, Mux(to_wait, entries(wait_id).hptw_resp, entries(enq_ptr).hptw_resp)) 5416979864eSXiaokun-Pei entries(enq_ptr).first_s2xlate_fault := false.B 5427299828dSXiaokun-Pei mem_resp_hit(enq_ptr) := to_mem_out || to_last_hptw_req 54392e3bfefSLemover } 5447797f035SbugGenerator 5457797f035SbugGenerator val enq_ptr_reg = RegNext(enq_ptr) 5465adc4829SYanqin Li val need_addr_check = GatedValidRegNext(enq_state === state_addr_check && io.in.fire && !flush) 5477274ec5cSpeixiaokun 5480214776eSpeixiaokun val hasHptwResp = ParallelOR(state.map(_ === state_hptw_resp)).asBool 5497274ec5cSpeixiaokun val hptw_resp_ptr_reg = RegNext(io.hptw.resp.bits.id) 550a664078aSpeixiaokun val hptw_need_addr_check = RegNext(hasHptwResp && io.hptw.resp.fire && !flush) && state(hptw_resp_ptr_reg) === state_addr_check 551d0de7e4aSpeixiaokun 552ce5f4200SGuanghui Hu val ptes = io.mem.resp.bits.value.asTypeOf(Vec(blockBits / XLEN, new PteBundle())) 5533211121aSXiaokun-Pei val gpaddr = MakeGPAddr(entries(hptw_resp_ptr_reg).ppn, getVpnn(entries(hptw_resp_ptr_reg).req_info.vpn, 0)) 55482e4705bSpeixiaokun val hptw_resp = entries(hptw_resp_ptr_reg).hptw_resp 555cda84113Speixiaokun val hpaddr = Cat(hptw_resp.genPPNS2(get_pn(gpaddr)), get_off(gpaddr)) 5564c0e0181SXiaokun-Pei val addr = RegEnable(MakeAddr(io.in.bits.ppn(ppnLen - 1, 0), getVpnn(io.in.bits.req_info.vpn, 0)), io.in.fire) 5577274ec5cSpeixiaokun io.pmp.req.valid := need_addr_check || hptw_need_addr_check 55882e4705bSpeixiaokun io.pmp.req.bits.addr := Mux(hptw_need_addr_check, hpaddr, addr) 5597797f035SbugGenerator io.pmp.req.bits.cmd := TlbCmd.read 5607797f035SbugGenerator io.pmp.req.bits.size := 3.U // TODO: fix it 5617797f035SbugGenerator val pmp_resp_valid = io.pmp.req.valid // same cycle 5627797f035SbugGenerator when (pmp_resp_valid) { 5637797f035SbugGenerator // NOTE: when pmp resp but state is not addr check, then the entry is dup with other entry, the state was changed before 5647797f035SbugGenerator // when dup with the req-ing entry, set to mem_waiting (above codes), and the ld must be false, so dontcare 5657274ec5cSpeixiaokun val ptr = Mux(hptw_need_addr_check, hptw_resp_ptr_reg, enq_ptr_reg); 5667797f035SbugGenerator val accessFault = io.pmp.resp.ld || io.pmp.resp.mmio 5677274ec5cSpeixiaokun entries(ptr).af := accessFault 5687274ec5cSpeixiaokun state(ptr) := Mux(accessFault, state_mem_out, state_mem_req) 5697797f035SbugGenerator } 5707797f035SbugGenerator 571935edac4STang Haojin when (mem_arb.io.out.fire) { 57292e3bfefSLemover for (i <- state.indices) { 573ec78ed87Speixiaokun when (state(i) =/= state_idle && state(i) =/= state_mem_out && state(i) =/= state_last_hptw_req && state(i) =/= state_last_hptw_resp 574ec78ed87Speixiaokun && entries(i).req_info.s2xlate === mem_arb.io.out.bits.req_info.s2xlate 575ec78ed87Speixiaokun && dup(entries(i).req_info.vpn, mem_arb.io.out.bits.req_info.vpn)) { 57692e3bfefSLemover // NOTE: "dup enq set state to mem_wait" -> "sending req set other dup entries to mem_wait" 57792e3bfefSLemover state(i) := state_mem_waiting 5782a1f48e7Speixiaokun entries(i).hptw_resp := entries(mem_arb.io.chosen).hptw_resp 57992e3bfefSLemover entries(i).wait_id := mem_arb.io.chosen 58092e3bfefSLemover } 58192e3bfefSLemover } 58292e3bfefSLemover } 583935edac4STang Haojin when (io.mem.resp.fire) { 58492e3bfefSLemover state.indices.map{i => 58592e3bfefSLemover when (state(i) === state_mem_waiting && io.mem.resp.bits.id === entries(i).wait_id) { 5864358f287Speixiaokun val req_paddr = MakeAddr(entries(i).ppn, getVpnn(entries(i).req_info.vpn, 0)) 5874358f287Speixiaokun val req_hpaddr = MakeAddr(entries(i).hptw_resp.genPPNS2(get_pn(req_paddr)), getVpnn(entries(i).req_info.vpn, 0)) 5884358f287Speixiaokun val index = Mux(entries(i).req_info.s2xlate === allStage, req_hpaddr, req_paddr)(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 589cf41a6eeSpeixiaokun state(i) := Mux(entries(i).req_info.s2xlate === allStage && !(ptes(index).isPf(2.U) || !ptes(index).isLeaf() || ptes(index).isAf()), state_last_hptw_req, state_mem_out) 590cf41a6eeSpeixiaokun mem_resp_hit(i) := true.B 5914c0e0181SXiaokun-Pei entries(i).ppn := ptes(index).getPPN() // for last stage 2 translation 592ad0d9d89Speixiaokun } 593ad0d9d89Speixiaokun } 594ad0d9d89Speixiaokun } 595ad0d9d89Speixiaokun 5963222d00fSpeixiaokun when (hyper_arb1.io.out.fire) { 597d0de7e4aSpeixiaokun for (i <- state.indices) { 5986b742a19SXiaokun-Pei when (state(i) === state_hptw_req && entries(i).ppn === hyper_arb1.io.out.bits.ppn && entries(i).req_info.s2xlate === allStage && hyper_arb1.io.chosen === i.U) { 599d0de7e4aSpeixiaokun state(i) := state_hptw_resp 600d0de7e4aSpeixiaokun entries(i).wait_id := hyper_arb1.io.chosen 601d0de7e4aSpeixiaokun } 602d0de7e4aSpeixiaokun } 603d0de7e4aSpeixiaokun } 604d0de7e4aSpeixiaokun 6053222d00fSpeixiaokun when (hyper_arb2.io.out.fire) { 606d0de7e4aSpeixiaokun for (i <- state.indices) { 6076b742a19SXiaokun-Pei when (state(i) === state_last_hptw_req && entries(i).ppn === hyper_arb2.io.out.bits.ppn && entries(i).req_info.s2xlate === allStage && hyper_arb2.io.chosen === i.U) { 608d0de7e4aSpeixiaokun state(i) := state_last_hptw_resp 609d0de7e4aSpeixiaokun entries(i).wait_id := hyper_arb2.io.chosen 610d0de7e4aSpeixiaokun } 611d0de7e4aSpeixiaokun } 612d0de7e4aSpeixiaokun } 613d0de7e4aSpeixiaokun 6143222d00fSpeixiaokun when (io.hptw.resp.fire) { 615d0de7e4aSpeixiaokun for (i <- state.indices) { 6162a1f48e7Speixiaokun when (state(i) === state_hptw_resp && io.hptw.resp.bits.id === entries(i).wait_id && io.hptw.resp.bits.h_resp.entry.tag === entries(i).ppn) { 61769f13e85SXiaokun-Pei when (io.hptw.resp.bits.h_resp.gaf || io.hptw.resp.bits.h_resp.gpf) { 61869f13e85SXiaokun-Pei state(i) := state_mem_out 61969f13e85SXiaokun-Pei entries(i).hptw_resp := io.hptw.resp.bits.h_resp 6206979864eSXiaokun-Pei entries(i).first_s2xlate_fault := io.hptw.resp.bits.h_resp.gaf || io.hptw.resp.bits.h_resp.gpf 62169f13e85SXiaokun-Pei }.otherwise{ // change the entry that is waiting hptw resp 622ec78ed87Speixiaokun val need_to_waiting_vec = state.indices.map(i => state(i) === state_mem_waiting && dup(entries(i).req_info.vpn, entries(io.hptw.resp.bits.id).req_info.vpn)) 6237f96e195Speixiaokun val waiting_index = ParallelMux(need_to_waiting_vec zip entries.map(_.wait_id)) 6247f96e195Speixiaokun state(i) := Mux(Cat(need_to_waiting_vec).orR, state_mem_waiting, state_addr_check) 625dc05c713Speixiaokun entries(i).hptw_resp := io.hptw.resp.bits.h_resp 6267f96e195Speixiaokun entries(i).wait_id := Mux(Cat(need_to_waiting_vec).orR, waiting_index, entries(i).wait_id) 6272a1f48e7Speixiaokun //To do: change the entry that is having the same hptw req 628d0de7e4aSpeixiaokun } 62969f13e85SXiaokun-Pei } 6302a1f48e7Speixiaokun when (state(i) === state_last_hptw_resp && io.hptw.resp.bits.id === entries(i).wait_id && io.hptw.resp.bits.h_resp.entry.tag === entries(i).ppn) { 631d0de7e4aSpeixiaokun state(i) := state_mem_out 632dc05c713Speixiaokun entries(i).hptw_resp := io.hptw.resp.bits.h_resp 6332a1f48e7Speixiaokun //To do: change the entry that is having the same hptw req 634d0de7e4aSpeixiaokun } 635d0de7e4aSpeixiaokun } 636d0de7e4aSpeixiaokun } 637935edac4STang Haojin when (io.out.fire) { 63892e3bfefSLemover assert(state(mem_ptr) === state_mem_out) 63992e3bfefSLemover state(mem_ptr) := state_idle 64092e3bfefSLemover } 64192e3bfefSLemover mem_resp_hit.map(a => when (a) { a := false.B } ) 64292e3bfefSLemover 6437797f035SbugGenerator when (io.cache.fire) { 6447797f035SbugGenerator state(cache_ptr) := state_idle 64592e3bfefSLemover } 6467797f035SbugGenerator XSError(io.out.fire && io.cache.fire && (mem_ptr === cache_ptr), "mem resp and cache fire at the same time at same entry") 64792e3bfefSLemover 64892e3bfefSLemover when (flush) { 64992e3bfefSLemover state.map(_ := state_idle) 65092e3bfefSLemover } 65192e3bfefSLemover 65292e3bfefSLemover io.in.ready := !full 65392e3bfefSLemover 654935edac4STang Haojin io.out.valid := ParallelOR(is_having).asBool 65592e3bfefSLemover io.out.bits.req_info := entries(mem_ptr).req_info 65692e3bfefSLemover io.out.bits.id := mem_ptr 65792e3bfefSLemover io.out.bits.af := entries(mem_ptr).af 658dc05c713Speixiaokun io.out.bits.h_resp := entries(mem_ptr).hptw_resp 6596979864eSXiaokun-Pei io.out.bits.first_s2xlate_fault := entries(mem_ptr).first_s2xlate_fault 660d0de7e4aSpeixiaokun 66183d93d53Speixiaokun val hptw_req_arb = Module(new Arbiter(new Bundle{ 66283d93d53Speixiaokun val source = UInt(bSourceWidth.W) 66383d93d53Speixiaokun val id = UInt(log2Up(l2tlbParams.llptwsize).W) 6644c0e0181SXiaokun-Pei val ppn = UInt(gvpnLen.W) 66583d93d53Speixiaokun } , 2)) 66683d93d53Speixiaokun // first stage 2 translation 66783d93d53Speixiaokun hptw_req_arb.io.in(0).valid := hyper_arb1.io.out.valid 66883d93d53Speixiaokun hptw_req_arb.io.in(0).bits.source := hyper_arb1.io.out.bits.req_info.source 66983d93d53Speixiaokun hptw_req_arb.io.in(0).bits.ppn := hyper_arb1.io.out.bits.ppn 67083d93d53Speixiaokun hptw_req_arb.io.in(0).bits.id := hyper_arb1.io.chosen 6712a1f48e7Speixiaokun hyper_arb1.io.out.ready := hptw_req_arb.io.in(0).ready 67283d93d53Speixiaokun // last stage 2 translation 67383d93d53Speixiaokun hptw_req_arb.io.in(1).valid := hyper_arb2.io.out.valid 67483d93d53Speixiaokun hptw_req_arb.io.in(1).bits.source := hyper_arb2.io.out.bits.req_info.source 67583d93d53Speixiaokun hptw_req_arb.io.in(1).bits.ppn := hyper_arb2.io.out.bits.ppn 67683d93d53Speixiaokun hptw_req_arb.io.in(1).bits.id := hyper_arb2.io.chosen 6772a1f48e7Speixiaokun hyper_arb2.io.out.ready := hptw_req_arb.io.in(1).ready 67883d93d53Speixiaokun hptw_req_arb.io.out.ready := io.hptw.req.ready 6792a1f48e7Speixiaokun io.hptw.req.valid := hptw_req_arb.io.out.fire && !flush 68083d93d53Speixiaokun io.hptw.req.bits.gvpn := hptw_req_arb.io.out.bits.ppn 68183d93d53Speixiaokun io.hptw.req.bits.id := hptw_req_arb.io.out.bits.id 68283d93d53Speixiaokun io.hptw.req.bits.source := hptw_req_arb.io.out.bits.source 68392e3bfefSLemover 68492e3bfefSLemover io.mem.req.valid := mem_arb.io.out.valid && !flush 685dc05c713Speixiaokun val mem_paddr = MakeAddr(mem_arb.io.out.bits.ppn, getVpnn(mem_arb.io.out.bits.req_info.vpn, 0)) 686cda84113Speixiaokun val mem_hpaddr = MakeAddr(mem_arb.io.out.bits.hptw_resp.genPPNS2(get_pn(mem_paddr)), getVpnn(mem_arb.io.out.bits.req_info.vpn, 0)) 6876b742a19SXiaokun-Pei io.mem.req.bits.addr := Mux(mem_arb.io.out.bits.req_info.s2xlate === allStage, mem_hpaddr, mem_paddr) 68892e3bfefSLemover io.mem.req.bits.id := mem_arb.io.chosen 68983d93d53Speixiaokun io.mem.req.bits.hptw_bypassed := false.B 69092e3bfefSLemover mem_arb.io.out.ready := io.mem.req.ready 691933ec998Speixiaokun val mem_refill_id = RegNext(io.mem.resp.bits.id(log2Up(l2tlbParams.llptwsize)-1, 0)) 692933ec998Speixiaokun io.mem.refill := entries(mem_refill_id).req_info 6934ed5afbdSXiaokun-Pei io.mem.refill.s2xlate := entries(mem_refill_id).req_info.s2xlate 69492e3bfefSLemover io.mem.buffer_it := mem_resp_hit 69592e3bfefSLemover io.mem.enq_ptr := enq_ptr 69692e3bfefSLemover 6977797f035SbugGenerator io.cache.valid := Cat(is_cache).orR 6987797f035SbugGenerator io.cache.bits := ParallelMux(is_cache, entries.map(_.req_info)) 6997797f035SbugGenerator 700935edac4STang Haojin XSPerfAccumulate("llptw_in_count", io.in.fire) 70192e3bfefSLemover XSPerfAccumulate("llptw_in_block", io.in.valid && !io.in.ready) 70292e3bfefSLemover for (i <- 0 until 7) { 703935edac4STang Haojin XSPerfAccumulate(s"enq_state${i}", io.in.fire && enq_state === i.U) 70492e3bfefSLemover } 70592e3bfefSLemover for (i <- 0 until (l2tlbParams.llptwsize + 1)) { 70692e3bfefSLemover XSPerfAccumulate(s"util${i}", PopCount(is_emptys.map(!_)) === i.U) 70792e3bfefSLemover XSPerfAccumulate(s"mem_util${i}", PopCount(is_mems) === i.U) 70892e3bfefSLemover XSPerfAccumulate(s"waiting_util${i}", PopCount(is_waiting) === i.U) 70992e3bfefSLemover } 710935edac4STang Haojin XSPerfAccumulate("mem_count", io.mem.req.fire) 71192e3bfefSLemover XSPerfAccumulate("mem_cycle", PopCount(is_waiting) =/= 0.U) 71292e3bfefSLemover XSPerfAccumulate("blocked_in", io.in.valid && !io.in.ready) 71392e3bfefSLemover 71492e3bfefSLemover for (i <- 0 until l2tlbParams.llptwsize) { 71592e3bfefSLemover TimeOutAssert(state(i) =/= state_idle, timeOutThreshold, s"missqueue time out no out ${i}") 71692e3bfefSLemover } 71792e3bfefSLemover 71892e3bfefSLemover val perfEvents = Seq( 719935edac4STang Haojin ("tlbllptw_incount ", io.in.fire ), 72092e3bfefSLemover ("tlbllptw_inblock ", io.in.valid && !io.in.ready), 721935edac4STang Haojin ("tlbllptw_memcount ", io.mem.req.fire ), 72292e3bfefSLemover ("tlbllptw_memcycle ", PopCount(is_waiting) ), 72392e3bfefSLemover ) 72492e3bfefSLemover generatePerfEvent() 72592e3bfefSLemover} 726d0de7e4aSpeixiaokun 727d0de7e4aSpeixiaokun/*========================= HPTW ==============================*/ 728d0de7e4aSpeixiaokun 729d0de7e4aSpeixiaokun/** HPTW : Hypervisor Page Table Walker 730d0de7e4aSpeixiaokun * the page walker take the virtual machine's page walk. 731d0de7e4aSpeixiaokun * guest physical address translation, guest physical address -> host physical address 732d0de7e4aSpeixiaokun **/ 733d0de7e4aSpeixiaokunclass HPTWIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 734d0de7e4aSpeixiaokun val req = Flipped(DecoupledIO(new Bundle { 735eb4bf3f2Speixiaokun val source = UInt(bSourceWidth.W) 736d0de7e4aSpeixiaokun val id = UInt(log2Up(l2tlbParams.llptwsize).W) 73782978df9Speixiaokun val gvpn = UInt(vpnLen.W) 7386315ba2aSpeixiaokun val ppn = UInt(ppnLen.W) 739d0de7e4aSpeixiaokun val l1Hit = Bool() 740d0de7e4aSpeixiaokun val l2Hit = Bool() 74183d93d53Speixiaokun val bypassed = Bool() // if bypass, don't refill 742d0de7e4aSpeixiaokun })) 743c2b430edSpeixiaokun val resp = DecoupledIO(new Bundle { 744eb4bf3f2Speixiaokun val source = UInt(bSourceWidth.W) 745d0de7e4aSpeixiaokun val resp = Output(new HptwResp()) 746d0de7e4aSpeixiaokun val id = Output(UInt(bMemID.W)) 747d0de7e4aSpeixiaokun }) 748d0de7e4aSpeixiaokun 749d0de7e4aSpeixiaokun val mem = new Bundle { 750d0de7e4aSpeixiaokun val req = DecoupledIO(new L2TlbMemReqBundle()) 751d0de7e4aSpeixiaokun val resp = Flipped(ValidIO(UInt(XLEN.W))) 752d0de7e4aSpeixiaokun val mask = Input(Bool()) 753d0de7e4aSpeixiaokun } 754d0de7e4aSpeixiaokun val refill = Output(new Bundle { 755d0de7e4aSpeixiaokun val req_info = new L2TlbInnerBundle() 756d0de7e4aSpeixiaokun val level = UInt(log2Up(Level).W) 757d0de7e4aSpeixiaokun }) 758d0de7e4aSpeixiaokun val pmp = new Bundle { 759d0de7e4aSpeixiaokun val req = ValidIO(new PMPReqBundle()) 760d0de7e4aSpeixiaokun val resp = Flipped(new PMPRespBundle()) 761d0de7e4aSpeixiaokun } 762d0de7e4aSpeixiaokun} 763d0de7e4aSpeixiaokun 764d0de7e4aSpeixiaokunclass HPTW()(implicit p: Parameters) extends XSModule with HasPtwConst { 765d0de7e4aSpeixiaokun val io = IO(new HPTWIO) 766d0de7e4aSpeixiaokun val hgatp = io.csr.hgatp 767d0de7e4aSpeixiaokun val sfence = io.sfence 7681ae5db63SXiaokun-Pei val flush = sfence.valid || hgatp.changed || io.csr.satp.changed || io.csr.vsatp.changed 769d0de7e4aSpeixiaokun 770d0de7e4aSpeixiaokun val level = RegInit(0.U(log2Up(Level).W)) 771d0de7e4aSpeixiaokun val gpaddr = Reg(UInt(GPAddrBits.W)) 7724c4af37cSpeixiaokun val req_ppn = Reg(UInt(ppnLen.W)) 773d0de7e4aSpeixiaokun val vpn = gpaddr(GPAddrBits-1, offLen) 774d0de7e4aSpeixiaokun val levelNext = level + 1.U 775d0de7e4aSpeixiaokun val l1Hit = Reg(Bool()) 776d0de7e4aSpeixiaokun val l2Hit = Reg(Bool()) 77783d93d53Speixiaokun val bypassed = Reg(Bool()) 778b24e0a78Speixiaokun val pg_base = MakeGPAddr(hgatp.ppn, getGVpnn(vpn, 2.U)) // for l0 779d0de7e4aSpeixiaokun// val pte = io.mem.resp.bits.MergeRespToPte() 780d0de7e4aSpeixiaokun val pte = io.mem.resp.bits.asTypeOf(new PteBundle().cloneType) 7814c4af37cSpeixiaokun val ppn_l1 = Mux(l1Hit, req_ppn, pte.ppn) 7824c4af37cSpeixiaokun val ppn_l2 = Mux(l2Hit, req_ppn, pte.ppn) 7836315ba2aSpeixiaokun val ppn = Mux(level === 1.U, ppn_l1, ppn_l2) //for l1 and l2 7846315ba2aSpeixiaokun val p_pte = MakeAddr(ppn, getVpnn(vpn, 2.U - level)) 785d0de7e4aSpeixiaokun val mem_addr = Mux(level === 0.U, pg_base, p_pte) 786d0de7e4aSpeixiaokun 787d0de7e4aSpeixiaokun //s/w register 788d0de7e4aSpeixiaokun val s_pmp_check = RegInit(true.B) 789d0de7e4aSpeixiaokun val s_mem_req = RegInit(true.B) 790d0de7e4aSpeixiaokun val w_mem_resp = RegInit(true.B) 791d0de7e4aSpeixiaokun val idle = RegInit(true.B) 79203c1129fSpeixiaokun val mem_addr_update = RegInit(false.B) 793d0de7e4aSpeixiaokun val finish = WireInit(false.B) 794d0de7e4aSpeixiaokun 795d0de7e4aSpeixiaokun val sent_to_pmp = !idle && (!s_pmp_check || mem_addr_update) && !finish 7966613a2d1SXiaokun-Pei val pageFault = pte.isPf(level) || (!pte.isLeaf() && level >= 2.U) 797d0de7e4aSpeixiaokun val accessFault = RegEnable(io.pmp.resp.ld || io.pmp.resp.mmio, sent_to_pmp) 798d0de7e4aSpeixiaokun 799d0de7e4aSpeixiaokun val ppn_af = pte.isAf() 800d0de7e4aSpeixiaokun val find_pte = pte.isLeaf() || ppn_af || pageFault 801d0de7e4aSpeixiaokun 802d0de7e4aSpeixiaokun val resp_valid = !idle && mem_addr_update && ((w_mem_resp && find_pte) || (s_pmp_check && accessFault)) 803d0de7e4aSpeixiaokun val id = Reg(UInt(log2Up(l2tlbParams.llptwsize).W)) 8043222d00fSpeixiaokun val source = RegEnable(io.req.bits.source, io.req.fire) 805eb4bf3f2Speixiaokun 806d0de7e4aSpeixiaokun io.req.ready := idle 807eb4bf3f2Speixiaokun val resp = Wire(new HptwResp()) 808d0de7e4aSpeixiaokun resp.apply(pageFault && !accessFault && !ppn_af, accessFault || ppn_af, level, pte, vpn, hgatp.asid) 809d0de7e4aSpeixiaokun io.resp.valid := resp_valid 810d0de7e4aSpeixiaokun io.resp.bits.id := id 811d0de7e4aSpeixiaokun io.resp.bits.resp := resp 812eb4bf3f2Speixiaokun io.resp.bits.source := source 813d0de7e4aSpeixiaokun 814d0de7e4aSpeixiaokun io.pmp.req.valid := DontCare 815d0de7e4aSpeixiaokun io.pmp.req.bits.addr := mem_addr 816d0de7e4aSpeixiaokun io.pmp.req.bits.size := 3.U 817d0de7e4aSpeixiaokun io.pmp.req.bits.cmd := TlbCmd.read 818d0de7e4aSpeixiaokun 819d0de7e4aSpeixiaokun io.mem.req.valid := !s_mem_req && !io.mem.mask && !accessFault && s_pmp_check 820d0de7e4aSpeixiaokun io.mem.req.bits.addr := mem_addr 821d0de7e4aSpeixiaokun io.mem.req.bits.id := HptwReqId.U(bMemID.W) 82283d93d53Speixiaokun io.mem.req.bits.hptw_bypassed := bypassed 823d0de7e4aSpeixiaokun 82482978df9Speixiaokun io.refill.req_info.vpn := vpn 825d0de7e4aSpeixiaokun io.refill.level := level 826eb4bf3f2Speixiaokun io.refill.req_info.source := source 827eb4bf3f2Speixiaokun io.refill.req_info.s2xlate := onlyStage2 828d0de7e4aSpeixiaokun when (idle){ 8293222d00fSpeixiaokun when(io.req.fire){ 83083d93d53Speixiaokun bypassed := io.req.bits.bypassed 831d0de7e4aSpeixiaokun level := Mux(io.req.bits.l2Hit, 2.U, Mux(io.req.bits.l1Hit, 1.U, 0.U)) 832d0de7e4aSpeixiaokun idle := false.B 833d0de7e4aSpeixiaokun gpaddr := Cat(io.req.bits.gvpn, 0.U(offLen.W)) 834d0de7e4aSpeixiaokun accessFault := false.B 835d0de7e4aSpeixiaokun s_pmp_check := false.B 836d0de7e4aSpeixiaokun id := io.req.bits.id 8374c4af37cSpeixiaokun req_ppn := io.req.bits.ppn 838d0de7e4aSpeixiaokun l1Hit := io.req.bits.l1Hit 839d0de7e4aSpeixiaokun l2Hit := io.req.bits.l2Hit 840d0de7e4aSpeixiaokun } 841d0de7e4aSpeixiaokun } 842d0de7e4aSpeixiaokun 843d0de7e4aSpeixiaokun when(sent_to_pmp && !mem_addr_update){ 844d0de7e4aSpeixiaokun s_mem_req := false.B 845d0de7e4aSpeixiaokun s_pmp_check := true.B 846d0de7e4aSpeixiaokun } 847d0de7e4aSpeixiaokun 848d0de7e4aSpeixiaokun when(accessFault && !idle){ 849d0de7e4aSpeixiaokun s_pmp_check := true.B 850d0de7e4aSpeixiaokun s_mem_req := true.B 851d0de7e4aSpeixiaokun w_mem_resp := true.B 852d0de7e4aSpeixiaokun mem_addr_update := true.B 853d0de7e4aSpeixiaokun } 854d0de7e4aSpeixiaokun 8553222d00fSpeixiaokun when(io.mem.req.fire){ 856d0de7e4aSpeixiaokun s_mem_req := true.B 857d0de7e4aSpeixiaokun w_mem_resp := false.B 858d0de7e4aSpeixiaokun } 859d0de7e4aSpeixiaokun 8603222d00fSpeixiaokun when(io.mem.resp.fire && !w_mem_resp){ 861d0de7e4aSpeixiaokun w_mem_resp := true.B 862d0de7e4aSpeixiaokun mem_addr_update := true.B 863d0de7e4aSpeixiaokun } 864d0de7e4aSpeixiaokun 865d0de7e4aSpeixiaokun when(mem_addr_update){ 866d0de7e4aSpeixiaokun when(!(find_pte || accessFault)){ 867d0de7e4aSpeixiaokun level := levelNext 868d0de7e4aSpeixiaokun s_mem_req := false.B 869d0de7e4aSpeixiaokun mem_addr_update := false.B 870d0de7e4aSpeixiaokun }.elsewhen(resp_valid){ 8713222d00fSpeixiaokun when(io.resp.fire){ 872d0de7e4aSpeixiaokun idle := true.B 873d0de7e4aSpeixiaokun mem_addr_update := false.B 874d0de7e4aSpeixiaokun accessFault := false.B 875d0de7e4aSpeixiaokun } 876d0de7e4aSpeixiaokun finish := true.B 877d0de7e4aSpeixiaokun } 878d0de7e4aSpeixiaokun } 8795961467fSXiaokun-Pei when (flush) { 8805961467fSXiaokun-Pei idle := true.B 8815961467fSXiaokun-Pei s_pmp_check := true.B 8825961467fSXiaokun-Pei s_mem_req := true.B 8835961467fSXiaokun-Pei w_mem_resp := true.B 8845961467fSXiaokun-Pei accessFault := false.B 8855961467fSXiaokun-Pei mem_addr_update := false.B 8865961467fSXiaokun-Pei } 887d0de7e4aSpeixiaokun} 888