1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import xiangshan._ 23import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 24import utils._ 25import utility._ 26import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 27import freechips.rocketchip.tilelink._ 28import xiangshan.backend.fu.{PMPReqBundle, PMPRespBundle} 29 30/** Page Table Walk is divided into two parts 31 * One, PTW: page walk for pde, except for leaf entries, one by one 32 * Two, LLPTW: page walk for pte, only the leaf entries(4KB), in parallel 33 */ 34 35 36/** PTW : page table walker 37 * a finite state machine 38 * only take 1GB and 2MB page walks 39 * or in other words, except the last level(leaf) 40 **/ 41class PTWIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 42 val req = Flipped(DecoupledIO(new Bundle { 43 val req_info = new L2TlbInnerBundle() 44 val l3Hit = if (EnableSv48) Some(new Bool()) else None 45 val l2Hit = Bool() 46 val ppn = UInt(ptePPNLen.W) 47 val stage1Hit = Bool() 48 val stage1 = new PtwMergeResp 49 })) 50 val resp = DecoupledIO(new Bundle { 51 val source = UInt(bSourceWidth.W) 52 val s2xlate = UInt(2.W) 53 val resp = new PtwMergeResp 54 val h_resp = new HptwResp 55 }) 56 57 val llptw = DecoupledIO(new LLPTWInBundle()) 58 // NOTE: llptw change from "connect to llptw" to "connect to page cache" 59 // to avoid corner case that caused duplicate entries 60 61 val hptw = new Bundle { 62 val req = DecoupledIO(new Bundle { 63 val source = UInt(bSourceWidth.W) 64 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 65 val gvpn = UInt(ptePPNLen.W) 66 }) 67 val resp = Flipped(Valid(new Bundle { 68 val h_resp = Output(new HptwResp) 69 })) 70 } 71 val mem = new Bundle { 72 val req = DecoupledIO(new L2TlbMemReqBundle()) 73 val resp = Flipped(ValidIO(UInt(XLEN.W))) 74 val mask = Input(Bool()) 75 } 76 val pmp = new Bundle { 77 val req = ValidIO(new PMPReqBundle()) 78 val resp = Flipped(new PMPRespBundle()) 79 } 80 81 val refill = Output(new Bundle { 82 val req_info = new L2TlbInnerBundle() 83 val level = UInt(log2Up(Level + 1).W) 84 }) 85} 86 87class PTW()(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 88 val io = IO(new PTWIO) 89 val sfence = io.sfence 90 val mem = io.mem 91 val req_s2xlate = Reg(UInt(2.W)) 92 val enableS2xlate = req_s2xlate =/= noS2xlate 93 val onlyS1xlate = req_s2xlate === onlyStage1 94 val onlyS2xlate = req_s2xlate === onlyStage2 95 96 val satp = Wire(new TlbSatpBundle()) 97 when (io.req.fire) { 98 satp := Mux(io.req.bits.req_info.s2xlate =/= noS2xlate, io.csr.vsatp, io.csr.satp) 99 } .otherwise { 100 satp := Mux(enableS2xlate, io.csr.vsatp, io.csr.satp) 101 } 102 103 val mode = satp.mode 104 val hgatp = io.csr.hgatp 105 val flush = io.sfence.valid || io.csr.satp.changed || io.csr.vsatp.changed || io.csr.hgatp.changed 106 val s2xlate = enableS2xlate && !onlyS1xlate 107 val level = RegInit(3.U(log2Up(Level + 1).W)) 108 val af_level = RegInit(3.U(log2Up(Level + 1).W)) // access fault return this level 109 val gpf_level = RegInit(3.U(log2Up(Level + 1).W)) 110 val ppn = Reg(UInt(ptePPNLen.W)) 111 val vpn = Reg(UInt(vpnLen.W)) // vpn or gvpn(onlyS2xlate) 112 val levelNext = level - 1.U 113 val l3Hit = Reg(Bool()) 114 val l2Hit = Reg(Bool()) 115 val pte = mem.resp.bits.asTypeOf(new PteBundle()) 116 117 // s/w register 118 val s_pmp_check = RegInit(true.B) 119 val s_mem_req = RegInit(true.B) 120 val s_llptw_req = RegInit(true.B) 121 val w_mem_resp = RegInit(true.B) 122 val s_hptw_req = RegInit(true.B) 123 val w_hptw_resp = RegInit(true.B) 124 val s_last_hptw_req = RegInit(true.B) 125 val w_last_hptw_resp = RegInit(true.B) 126 // for updating "level" 127 val mem_addr_update = RegInit(false.B) 128 129 val idle = RegInit(true.B) 130 val finish = WireInit(false.B) 131 val sent_to_pmp = idle === false.B && (s_pmp_check === false.B || mem_addr_update) && !finish 132 133 val pageFault = pte.isPf(level) 134 val accessFault = RegEnable(io.pmp.resp.ld || io.pmp.resp.mmio, false.B, sent_to_pmp) 135 136 val hptw_pageFault = RegInit(false.B) 137 val hptw_accessFault = RegInit(false.B) 138 val last_s2xlate = RegInit(false.B) 139 val stage1Hit = RegEnable(io.req.bits.stage1Hit, io.req.fire) 140 val stage1 = RegEnable(io.req.bits.stage1, io.req.fire) 141 val hptw_resp_stage2 = Reg(Bool()) 142 143 val ppn_af = Mux(enableS2xlate, Mux(onlyS1xlate, pte.isAf() && !pte.isStage1Gpf(io.csr.vsatp.mode), false.B), pte.isAf()) // In two-stage address translation, stage 1 ppn is a vpn for host, so don't need to check ppn_high 144 val find_pte = pte.isLeaf() || ppn_af || pageFault 145 val to_find_pte = level === 1.U && find_pte === false.B 146 val source = RegEnable(io.req.bits.req_info.source, io.req.fire) 147 148 val l3addr = Wire(UInt(PAddrBits.W)) 149 val l2addr = Wire(UInt(PAddrBits.W)) 150 val l1addr = Wire(UInt(PAddrBits.W)) 151 val mem_addr = Wire(UInt(PAddrBits.W)) 152 153 l3addr := MakeAddr(satp.ppn, getVpnn(vpn, 3)) 154 if (EnableSv48) { 155 when (mode === Sv48) { 156 l2addr := MakeAddr(Mux(l3Hit, ppn, pte.getPPN()), getVpnn(vpn, 2)) 157 } .otherwise { 158 l2addr := MakeAddr(satp.ppn, getVpnn(vpn, 2)) 159 } 160 } else { 161 l2addr := MakeAddr(satp.ppn, getVpnn(vpn, 2)) 162 } 163 l1addr := MakeAddr(Mux(l2Hit, ppn, pte.getPPN()), getVpnn(vpn, 1)) 164 mem_addr := Mux(af_level === 3.U, l3addr, Mux(af_level === 2.U, l2addr, l1addr)) 165 166 val hptw_resp = Reg(new HptwResp) 167 val gpaddr = MuxCase(mem_addr, Seq( 168 stage1Hit -> Cat(stage1.genPPN(), 0.U(offLen.W)), 169 onlyS2xlate -> Cat(vpn, 0.U(offLen.W)), 170 !s_last_hptw_req -> Cat(MuxLookup(level, pte.getPPN())(Seq( 171 3.U -> Cat(pte.getPPN()(ptePPNLen - 1, vpnnLen * 3), vpn(vpnnLen * 3 - 1, 0)), 172 2.U -> Cat(pte.getPPN()(ptePPNLen - 1, vpnnLen * 2), vpn(vpnnLen * 2 - 1, 0)), 173 1.U -> Cat(pte.getPPN()(ptePPNLen - 1, vpnnLen), vpn(vpnnLen - 1, 0) 174 ))), 175 0.U(offLen.W)) 176 )) 177 val gvpn_gpf = Mux(enableS2xlate && io.csr.hgatp.mode === Sv39x4, gpaddr(gpaddr.getWidth - 1, GPAddrBitsSv39x4) =/= 0.U, Mux(enableS2xlate && io.csr.hgatp.mode === Sv48x4, gpaddr(gpaddr.getWidth - 1, GPAddrBitsSv48x4) =/= 0.U, false.B)) 178 val check_g_perm_fail = RegInit(false.B) 179 val guestFault = hptw_pageFault || hptw_accessFault || check_g_perm_fail || gvpn_gpf 180 val hpaddr = Cat(hptw_resp.genPPNS2(get_pn(gpaddr)), get_off(gpaddr)) 181 val fake_h_resp = 0.U.asTypeOf(new HptwResp) 182 fake_h_resp.entry.tag := get_pn(gpaddr) 183 fake_h_resp.entry.vmid.map(_ := io.csr.hgatp.vmid) 184 fake_h_resp.gpf := true.B 185 186 val pte_valid = RegInit(false.B) // avoid l1tlb pf from stage1 when gpf happens in the first s2xlate in PTW 187 val fake_pte = 0.U.asTypeOf(new PteBundle()) 188 fake_pte.perm.v := false.B // tell L1TLB this is fake pte 189 fake_pte.perm.r := true.B 190 fake_pte.perm.w := true.B 191 fake_pte.perm.x := true.B 192 fake_pte.perm.a := true.B 193 fake_pte.perm.d := true.B 194 fake_pte.ppn := ppn(ppnLen - 1, 0) 195 fake_pte.ppn_high := ppn(ptePPNLen - 1, ppnLen) 196 197 io.req.ready := idle 198 val ptw_resp = Wire(new PtwMergeResp) 199 ptw_resp.apply(Mux(pte_valid, pageFault && !accessFault && !ppn_af, false.B), accessFault || ppn_af, Mux(accessFault, af_level, Mux(guestFault, gpf_level, level)), Mux(pte_valid, pte, fake_pte), vpn, satp.asid, hgatp.vmid, vpn(sectortlbwidth - 1, 0), not_super = false) 200 201 val normal_resp = idle === false.B && mem_addr_update && !last_s2xlate && (guestFault || (w_mem_resp && find_pte) || (s_pmp_check && accessFault) || onlyS2xlate ) 202 val stageHit_resp = idle === false.B && hptw_resp_stage2 203 io.resp.valid := Mux(stage1Hit, stageHit_resp, normal_resp) 204 io.resp.bits.source := source 205 io.resp.bits.resp := Mux(stage1Hit || (l3Hit || l2Hit) && guestFault && !pte_valid, stage1, ptw_resp) 206 io.resp.bits.h_resp := Mux(gvpn_gpf, fake_h_resp, hptw_resp) 207 io.resp.bits.s2xlate := req_s2xlate 208 209 io.llptw.valid := s_llptw_req === false.B && to_find_pte && !accessFault && !guestFault 210 io.llptw.bits.req_info.source := source 211 io.llptw.bits.req_info.vpn := vpn 212 io.llptw.bits.req_info.s2xlate := req_s2xlate 213 io.llptw.bits.ppn := DontCare 214 215 io.pmp.req.valid := DontCare // samecycle, do not use valid 216 io.pmp.req.bits.addr := Mux(s2xlate, hpaddr, mem_addr) 217 io.pmp.req.bits.size := 3.U // TODO: fix it 218 io.pmp.req.bits.cmd := TlbCmd.read 219 220 mem.req.valid := s_mem_req === false.B && !mem.mask && !accessFault && s_pmp_check 221 mem.req.bits.addr := Mux(s2xlate, hpaddr, mem_addr) 222 mem.req.bits.id := FsmReqID.U(bMemID.W) 223 mem.req.bits.hptw_bypassed := false.B 224 225 io.refill.req_info.s2xlate := req_s2xlate 226 io.refill.req_info.vpn := vpn 227 io.refill.level := level 228 io.refill.req_info.source := source 229 230 io.hptw.req.valid := !s_hptw_req || !s_last_hptw_req 231 io.hptw.req.bits.id := FsmReqID.U(bMemID.W) 232 io.hptw.req.bits.gvpn := get_pn(gpaddr) 233 io.hptw.req.bits.source := source 234 235 when (io.req.fire && io.req.bits.stage1Hit){ 236 idle := false.B 237 req_s2xlate := io.req.bits.req_info.s2xlate 238 s_last_hptw_req := false.B 239 hptw_resp_stage2 := false.B 240 last_s2xlate := false.B 241 check_g_perm_fail := false.B 242 hptw_pageFault := false.B 243 hptw_accessFault := false.B 244 } 245 246 when (io.resp.fire && stage1Hit){ 247 idle := true.B 248 } 249 250 when (io.req.fire && !io.req.bits.stage1Hit){ 251 val req = io.req.bits 252 if (EnableSv48) { 253 when (mode === Sv48) { 254 level := Mux(req.l2Hit, 1.U, Mux(req.l3Hit.get, 2.U, 3.U)) 255 af_level := Mux(req.l2Hit, 1.U, Mux(req.l3Hit.get, 2.U, 3.U)) 256 gpf_level := Mux(req.l2Hit, 2.U, Mux(req.l3Hit.get, 3.U, 0.U)) 257 ppn := Mux(req.l2Hit || req.l3Hit.get, io.req.bits.ppn, satp.ppn) 258 l3Hit := req.l3Hit.get 259 } .otherwise { 260 level := Mux(req.l2Hit, 1.U, 2.U) 261 af_level := Mux(req.l2Hit, 1.U, 2.U) 262 gpf_level := 0.U 263 ppn := Mux(req.l2Hit, io.req.bits.ppn, satp.ppn) 264 l3Hit := false.B 265 } 266 } else { 267 level := Mux(req.l2Hit, 1.U, 2.U) 268 af_level := Mux(req.l2Hit, 1.U, 2.U) 269 gpf_level := 0.U 270 ppn := Mux(req.l2Hit, io.req.bits.ppn, satp.ppn) 271 l3Hit := false.B 272 } 273 vpn := io.req.bits.req_info.vpn 274 l2Hit := req.l2Hit 275 accessFault := false.B 276 idle := false.B 277 hptw_pageFault := false.B 278 hptw_accessFault := false.B 279 pte_valid := false.B 280 check_g_perm_fail := false.B 281 req_s2xlate := io.req.bits.req_info.s2xlate 282 when(io.req.bits.req_info.s2xlate === onlyStage2){ 283 val onlys2_gpaddr = Cat(io.req.bits.req_info.vpn, 0.U(offLen.W)) // is 50 bits, don't need to check high bits when sv48x4 is enabled 284 val check_gpa_high_fail = Mux(io.req.bits.req_info.s2xlate === onlyStage2 && io.csr.hgatp.mode === Sv39x4, onlys2_gpaddr(onlys2_gpaddr.getWidth - 1, GPAddrBitsSv39x4) =/= 0.U, false.B) 285 last_s2xlate := false.B 286 when(check_gpa_high_fail){ 287 mem_addr_update := true.B 288 }.otherwise{ 289 s_last_hptw_req := false.B 290 } 291 }.elsewhen(io.req.bits.req_info.s2xlate === allStage){ 292 last_s2xlate := true.B 293 s_hptw_req := false.B 294 }.otherwise { 295 last_s2xlate := false.B 296 s_pmp_check := false.B 297 } 298 } 299 300 when(io.hptw.req.fire && s_hptw_req === false.B){ 301 s_hptw_req := true.B 302 w_hptw_resp := false.B 303 } 304 305 when(io.hptw.resp.fire && w_hptw_resp === false.B) { 306 hptw_pageFault := io.hptw.resp.bits.h_resp.gpf 307 hptw_accessFault := io.hptw.resp.bits.h_resp.gaf 308 hptw_resp := io.hptw.resp.bits.h_resp 309 w_hptw_resp := true.B 310 val g_perm_fail = !io.hptw.resp.bits.h_resp.entry.perm.get.r && !(io.csr.priv.mxr && io.hptw.resp.bits.h_resp.entry.perm.get.x) 311 check_g_perm_fail := g_perm_fail 312 when(!(g_perm_fail || io.hptw.resp.bits.h_resp.gpf || io.hptw.resp.bits.h_resp.gaf)) { 313 s_pmp_check := false.B 314 } 315 } 316 317 when(io.hptw.req.fire && s_last_hptw_req === false.B) { 318 w_last_hptw_resp := false.B 319 s_last_hptw_req := true.B 320 } 321 322 when (io.hptw.resp.fire && w_last_hptw_resp === false.B && stage1Hit){ 323 w_last_hptw_resp := true.B 324 hptw_resp_stage2 := true.B 325 hptw_resp := io.hptw.resp.bits.h_resp 326 } 327 328 when(io.hptw.resp.fire && w_last_hptw_resp === false.B && !stage1Hit){ 329 hptw_pageFault := io.hptw.resp.bits.h_resp.gpf 330 hptw_accessFault := io.hptw.resp.bits.h_resp.gaf 331 hptw_resp := io.hptw.resp.bits.h_resp 332 w_last_hptw_resp := true.B 333 mem_addr_update := true.B 334 last_s2xlate := false.B 335 } 336 337 when(sent_to_pmp && mem_addr_update === false.B){ 338 s_mem_req := false.B 339 s_pmp_check := true.B 340 } 341 342 when(accessFault && idle === false.B){ 343 s_pmp_check := true.B 344 s_mem_req := true.B 345 w_mem_resp := true.B 346 s_llptw_req := true.B 347 s_hptw_req := true.B 348 w_hptw_resp := true.B 349 s_last_hptw_req := true.B 350 w_last_hptw_resp := true.B 351 mem_addr_update := true.B 352 last_s2xlate := false.B 353 } 354 355 when(guestFault && idle === false.B){ 356 s_pmp_check := true.B 357 s_mem_req := true.B 358 w_mem_resp := true.B 359 s_llptw_req := true.B 360 s_hptw_req := true.B 361 w_hptw_resp := true.B 362 s_last_hptw_req := true.B 363 w_last_hptw_resp := true.B 364 mem_addr_update := true.B 365 last_s2xlate := false.B 366 } 367 368 when (mem.req.fire){ 369 s_mem_req := true.B 370 w_mem_resp := false.B 371 } 372 373 when(mem.resp.fire && w_mem_resp === false.B){ 374 w_mem_resp := true.B 375 af_level := af_level - 1.U 376 s_llptw_req := false.B 377 mem_addr_update := true.B 378 gpf_level := Mux(mode === Sv39 && !pte_valid && !(l3Hit || l2Hit), gpf_level - 2.U, gpf_level - 1.U) 379 pte_valid := true.B 380 } 381 382 when(mem_addr_update){ 383 when(level >= 2.U && !onlyS2xlate && !(guestFault || find_pte || accessFault)) { 384 level := levelNext 385 when(s2xlate){ 386 s_hptw_req := false.B 387 }.otherwise{ 388 s_mem_req := false.B 389 } 390 s_llptw_req := true.B 391 mem_addr_update := false.B 392 }.elsewhen(io.llptw.valid){ 393 when(io.llptw.fire) { 394 idle := true.B 395 s_llptw_req := true.B 396 mem_addr_update := false.B 397 last_s2xlate := false.B 398 } 399 finish := true.B 400 }.elsewhen(s2xlate && last_s2xlate === true.B) { 401 when(accessFault || pageFault || ppn_af){ 402 last_s2xlate := false.B 403 }.otherwise{ 404 s_last_hptw_req := false.B 405 mem_addr_update := false.B 406 } 407 }.elsewhen(io.resp.valid){ 408 when(io.resp.fire) { 409 idle := true.B 410 s_llptw_req := true.B 411 mem_addr_update := false.B 412 accessFault := false.B 413 } 414 finish := true.B 415 } 416 } 417 418 419 when (flush) { 420 idle := true.B 421 s_pmp_check := true.B 422 s_mem_req := true.B 423 s_llptw_req := true.B 424 w_mem_resp := true.B 425 accessFault := false.B 426 mem_addr_update := false.B 427 s_hptw_req := true.B 428 w_hptw_resp := true.B 429 s_last_hptw_req := true.B 430 w_last_hptw_resp := true.B 431 check_g_perm_fail := false.B 432 } 433 434 435 XSDebug(p"[ptw] level:${level} notFound:${pageFault}\n") 436 437 // perf 438 XSPerfAccumulate("fsm_count", io.req.fire) 439 for (i <- 0 until PtwWidth) { 440 XSPerfAccumulate(s"fsm_count_source${i}", io.req.fire && io.req.bits.req_info.source === i.U) 441 } 442 XSPerfAccumulate("fsm_busy", !idle) 443 XSPerfAccumulate("fsm_idle", idle) 444 XSPerfAccumulate("resp_blocked", io.resp.valid && !io.resp.ready) 445 XSPerfAccumulate("ptw_ppn_af", io.resp.fire && ppn_af) 446 XSPerfAccumulate("mem_count", mem.req.fire) 447 XSPerfAccumulate("mem_cycle", BoolStopWatch(mem.req.fire, mem.resp.fire, true)) 448 XSPerfAccumulate("mem_blocked", mem.req.valid && !mem.req.ready) 449 450 TimeOutAssert(!idle, timeOutThreshold, "page table walker time out") 451 452 val perfEvents = Seq( 453 ("fsm_count ", io.req.fire ), 454 ("fsm_busy ", !idle ), 455 ("fsm_idle ", idle ), 456 ("resp_blocked ", io.resp.valid && !io.resp.ready ), 457 ("mem_count ", mem.req.fire ), 458 ("mem_cycle ", BoolStopWatch(mem.req.fire, mem.resp.fire, true)), 459 ("mem_blocked ", mem.req.valid && !mem.req.ready ), 460 ) 461 generatePerfEvent() 462} 463 464/*========================= LLPTW ==============================*/ 465 466/** LLPTW : Last Level Page Table Walker 467 * the page walker that only takes 4KB(last level) page walk. 468 **/ 469 470class LLPTWInBundle(implicit p: Parameters) extends XSBundle with HasPtwConst { 471 val req_info = Output(new L2TlbInnerBundle()) 472 val ppn = Output(UInt(ptePPNLen.W)) 473} 474 475class LLPTWIO(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 476 val in = Flipped(DecoupledIO(new LLPTWInBundle())) 477 val out = DecoupledIO(new Bundle { 478 val req_info = Output(new L2TlbInnerBundle()) 479 val id = Output(UInt(bMemID.W)) 480 val h_resp = Output(new HptwResp) 481 val first_s2xlate_fault = Output(Bool()) // Whether the first stage 2 translation occurs pf/af 482 val af = Output(Bool()) 483 }) 484 val mem = new Bundle { 485 val req = DecoupledIO(new L2TlbMemReqBundle()) 486 val resp = Flipped(Valid(new Bundle { 487 val id = Output(UInt(log2Up(l2tlbParams.llptwsize).W)) 488 val value = Output(UInt(blockBits.W)) 489 })) 490 val enq_ptr = Output(UInt(log2Ceil(l2tlbParams.llptwsize).W)) 491 val buffer_it = Output(Vec(l2tlbParams.llptwsize, Bool())) 492 val refill = Output(new L2TlbInnerBundle()) 493 val req_mask = Input(Vec(l2tlbParams.llptwsize, Bool())) 494 val flush_latch = Input(Vec(l2tlbParams.llptwsize, Bool())) 495 } 496 val cache = DecoupledIO(new L2TlbInnerBundle()) 497 val pmp = new Bundle { 498 val req = Valid(new PMPReqBundle()) 499 val resp = Flipped(new PMPRespBundle()) 500 } 501 val hptw = new Bundle { 502 val req = DecoupledIO(new Bundle{ 503 val source = UInt(bSourceWidth.W) 504 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 505 val gvpn = UInt(ptePPNLen.W) 506 }) 507 val resp = Flipped(Valid(new Bundle { 508 val id = Output(UInt(log2Up(l2tlbParams.llptwsize).W)) 509 val h_resp = Output(new HptwResp) 510 })) 511 } 512} 513 514class LLPTWEntry(implicit p: Parameters) extends XSBundle with HasPtwConst { 515 val req_info = new L2TlbInnerBundle() 516 val ppn = UInt(ptePPNLen.W) 517 val wait_id = UInt(log2Up(l2tlbParams.llptwsize).W) 518 val af = Bool() 519 val hptw_resp = new HptwResp() 520 val first_s2xlate_fault = Output(Bool()) 521} 522 523 524class LLPTW(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 525 val io = IO(new LLPTWIO()) 526 val enableS2xlate = io.in.bits.req_info.s2xlate =/= noS2xlate 527 val satp = Mux(enableS2xlate, io.csr.vsatp, io.csr.satp) 528 529 val flush = io.sfence.valid || io.csr.satp.changed || io.csr.vsatp.changed || io.csr.hgatp.changed 530 val entries = RegInit(VecInit(Seq.fill(l2tlbParams.llptwsize)(0.U.asTypeOf(new LLPTWEntry())))) 531 val state_idle :: state_hptw_req :: state_hptw_resp :: state_addr_check :: state_mem_req :: state_mem_waiting :: state_mem_out :: state_last_hptw_req :: state_last_hptw_resp :: state_cache :: Nil = Enum(10) 532 val state = RegInit(VecInit(Seq.fill(l2tlbParams.llptwsize)(state_idle))) 533 534 val is_emptys = state.map(_ === state_idle) 535 val is_mems = state.map(_ === state_mem_req) 536 val is_waiting = state.map(_ === state_mem_waiting) 537 val is_having = state.map(_ === state_mem_out) 538 val is_cache = state.map(_ === state_cache) 539 val is_hptw_req = state.map(_ === state_hptw_req) 540 val is_last_hptw_req = state.map(_ === state_last_hptw_req) 541 val is_hptw_resp = state.map(_ === state_hptw_resp) 542 val is_last_hptw_resp = state.map(_ === state_last_hptw_resp) 543 544 val full = !ParallelOR(is_emptys).asBool 545 val enq_ptr = ParallelPriorityEncoder(is_emptys) 546 547 val mem_ptr = ParallelPriorityEncoder(is_having) // TODO: optimize timing, bad: entries -> ptr -> entry 548 val mem_arb = Module(new RRArbiterInit(new LLPTWEntry(), l2tlbParams.llptwsize)) 549 for (i <- 0 until l2tlbParams.llptwsize) { 550 mem_arb.io.in(i).bits := entries(i) 551 mem_arb.io.in(i).valid := is_mems(i) && !io.mem.req_mask(i) 552 } 553 554 // process hptw requests in serial 555 val hyper_arb1 = Module(new RRArbiterInit(new LLPTWEntry(), l2tlbParams.llptwsize)) 556 for (i <- 0 until l2tlbParams.llptwsize) { 557 hyper_arb1.io.in(i).bits := entries(i) 558 hyper_arb1.io.in(i).valid := is_hptw_req(i) && !(Cat(is_hptw_resp).orR) && !(Cat(is_last_hptw_resp).orR) 559 } 560 val hyper_arb2 = Module(new RRArbiterInit(new LLPTWEntry(), l2tlbParams.llptwsize)) 561 for(i <- 0 until l2tlbParams.llptwsize) { 562 hyper_arb2.io.in(i).bits := entries(i) 563 hyper_arb2.io.in(i).valid := is_last_hptw_req(i) && !(Cat(is_hptw_resp).orR) && !(Cat(is_last_hptw_resp).orR) 564 } 565 566 val cache_ptr = ParallelMux(is_cache, (0 until l2tlbParams.llptwsize).map(_.U(log2Up(l2tlbParams.llptwsize).W))) 567 568 // duplicate req 569 // to_wait: wait for the last to access mem, set to mem_resp 570 // to_cache: the last is back just right now, set to mem_cache 571 val dup_vec = state.indices.map(i => 572 dup(io.in.bits.req_info.vpn, entries(i).req_info.vpn) && io.in.bits.req_info.s2xlate === entries(i).req_info.s2xlate 573 ) 574 val dup_req_fire = mem_arb.io.out.fire && dup(io.in.bits.req_info.vpn, mem_arb.io.out.bits.req_info.vpn) && io.in.bits.req_info.s2xlate === mem_arb.io.out.bits.req_info.s2xlate // dup with the req fire entry 575 val dup_vec_wait = dup_vec.zip(is_waiting).map{case (d, w) => d && w} // dup with "mem_waiting" entries, sending mem req already 576 val dup_vec_having = dup_vec.zipWithIndex.map{case (d, i) => d && is_having(i)} // dup with the "mem_out" entry recv the data just now 577 val dup_vec_last_hptw = dup_vec.zipWithIndex.map{case (d, i) => d && (is_last_hptw_req(i) || is_last_hptw_resp(i))} 578 val wait_id = Mux(dup_req_fire, mem_arb.io.chosen, ParallelMux(dup_vec_wait zip entries.map(_.wait_id))) 579 val dup_wait_resp = io.mem.resp.fire && VecInit(dup_vec_wait)(io.mem.resp.bits.id) && !io.mem.flush_latch(io.mem.resp.bits.id) // dup with the entry that data coming next cycle 580 val to_wait = Cat(dup_vec_wait).orR || dup_req_fire 581 val to_mem_out = dup_wait_resp && ((entries(io.mem.resp.bits.id).req_info.s2xlate === noS2xlate) || (entries(io.mem.resp.bits.id).req_info.s2xlate === onlyStage1)) 582 val to_cache = Cat(dup_vec_having).orR || Cat(dup_vec_last_hptw).orR 583 val to_hptw_req = io.in.bits.req_info.s2xlate === allStage 584 val to_last_hptw_req = dup_wait_resp && entries(io.mem.resp.bits.id).req_info.s2xlate === allStage 585 val last_hptw_req_id = io.mem.resp.bits.id 586 val req_paddr = MakeAddr(io.in.bits.ppn(ppnLen-1, 0), getVpnn(io.in.bits.req_info.vpn, 0)) 587 val req_hpaddr = MakeAddr(entries(last_hptw_req_id).hptw_resp.genPPNS2(get_pn(req_paddr)), getVpnn(io.in.bits.req_info.vpn, 0)) 588 val index = Mux(entries(last_hptw_req_id).req_info.s2xlate === allStage, req_hpaddr, req_paddr)(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 589 val last_hptw_req_ppn = io.mem.resp.bits.value.asTypeOf(Vec(blockBits / XLEN, new PteBundle()))(index).getPPN() 590 XSError(RegNext(dup_req_fire && Cat(dup_vec_wait).orR, init = false.B), "mem req but some entries already waiting, should not happed") 591 592 XSError(io.in.fire && ((to_mem_out && to_cache) || (to_wait && to_cache)), "llptw enq, to cache conflict with to mem") 593 val mem_resp_hit = RegInit(VecInit(Seq.fill(l2tlbParams.llptwsize)(false.B))) 594 val enq_state_normal = MuxCase(state_addr_check, Seq( 595 to_mem_out -> state_mem_out, // same to the blew, but the mem resp now 596 to_last_hptw_req -> state_last_hptw_req, 597 to_wait -> state_mem_waiting, 598 to_cache -> state_cache, 599 to_hptw_req -> state_hptw_req 600 )) 601 val enq_state = Mux(from_pre(io.in.bits.req_info.source) && enq_state_normal =/= state_addr_check, state_idle, enq_state_normal) 602 when (io.in.fire) { 603 // if prefetch req does not need mem access, just give it up. 604 // so there will be at most 1 + FilterSize entries that needs re-access page cache 605 // so 2 + FilterSize is enough to avoid dead-lock 606 state(enq_ptr) := enq_state 607 entries(enq_ptr).req_info := io.in.bits.req_info 608 entries(enq_ptr).ppn := Mux(to_last_hptw_req, last_hptw_req_ppn, io.in.bits.ppn) 609 entries(enq_ptr).wait_id := Mux(to_wait, wait_id, enq_ptr) 610 entries(enq_ptr).af := false.B 611 entries(enq_ptr).hptw_resp := Mux(to_last_hptw_req, entries(last_hptw_req_id).hptw_resp, Mux(to_wait, entries(wait_id).hptw_resp, entries(enq_ptr).hptw_resp)) 612 entries(enq_ptr).first_s2xlate_fault := false.B 613 mem_resp_hit(enq_ptr) := to_mem_out || to_last_hptw_req 614 } 615 616 val enq_ptr_reg = RegNext(enq_ptr) 617 val need_addr_check = GatedValidRegNext(enq_state === state_addr_check && io.in.fire && !flush) 618 619 val hasHptwResp = ParallelOR(state.map(_ === state_hptw_resp)).asBool 620 val hptw_resp_ptr_reg = RegNext(io.hptw.resp.bits.id) 621 val hptw_need_addr_check = RegNext(hasHptwResp && io.hptw.resp.fire && !flush) && state(hptw_resp_ptr_reg) === state_addr_check 622 623 val ptes = io.mem.resp.bits.value.asTypeOf(Vec(blockBits / XLEN, new PteBundle())) 624 val gpaddr = MakeGPAddr(entries(hptw_resp_ptr_reg).ppn, getVpnn(entries(hptw_resp_ptr_reg).req_info.vpn, 0)) 625 val hptw_resp = entries(hptw_resp_ptr_reg).hptw_resp 626 val hpaddr = Cat(hptw_resp.genPPNS2(get_pn(gpaddr)), get_off(gpaddr)) 627 val addr = RegEnable(MakeAddr(io.in.bits.ppn(ppnLen - 1, 0), getVpnn(io.in.bits.req_info.vpn, 0)), io.in.fire) 628 io.pmp.req.valid := need_addr_check || hptw_need_addr_check 629 io.pmp.req.bits.addr := Mux(hptw_need_addr_check, hpaddr, addr) 630 io.pmp.req.bits.cmd := TlbCmd.read 631 io.pmp.req.bits.size := 3.U // TODO: fix it 632 val pmp_resp_valid = io.pmp.req.valid // same cycle 633 when (pmp_resp_valid) { 634 // NOTE: when pmp resp but state is not addr check, then the entry is dup with other entry, the state was changed before 635 // when dup with the req-ing entry, set to mem_waiting (above codes), and the ld must be false, so dontcare 636 val ptr = Mux(hptw_need_addr_check, hptw_resp_ptr_reg, enq_ptr_reg); 637 val accessFault = io.pmp.resp.ld || io.pmp.resp.mmio 638 entries(ptr).af := accessFault 639 state(ptr) := Mux(accessFault, state_mem_out, state_mem_req) 640 } 641 642 when (mem_arb.io.out.fire) { 643 for (i <- state.indices) { 644 when (state(i) =/= state_idle && state(i) =/= state_mem_out && state(i) =/= state_last_hptw_req && state(i) =/= state_last_hptw_resp 645 && entries(i).req_info.s2xlate === mem_arb.io.out.bits.req_info.s2xlate 646 && dup(entries(i).req_info.vpn, mem_arb.io.out.bits.req_info.vpn)) { 647 // NOTE: "dup enq set state to mem_wait" -> "sending req set other dup entries to mem_wait" 648 state(i) := state_mem_waiting 649 entries(i).hptw_resp := entries(mem_arb.io.chosen).hptw_resp 650 entries(i).wait_id := mem_arb.io.chosen 651 } 652 } 653 } 654 when (io.mem.resp.fire) { 655 state.indices.map{i => 656 when (state(i) === state_mem_waiting && io.mem.resp.bits.id === entries(i).wait_id) { 657 val req_paddr = MakeAddr(entries(i).ppn, getVpnn(entries(i).req_info.vpn, 0)) 658 val req_hpaddr = MakeAddr(entries(i).hptw_resp.genPPNS2(get_pn(req_paddr)), getVpnn(entries(i).req_info.vpn, 0)) 659 val index = Mux(entries(i).req_info.s2xlate === allStage, req_hpaddr, req_paddr)(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 660 state(i) := Mux(entries(i).req_info.s2xlate === allStage && !(ptes(index).isPf(0.U) || !ptes(index).isLeaf() || ptes(index).isAf() || ptes(index).isStage1Gpf(io.csr.vsatp.mode)) 661 , state_last_hptw_req, state_mem_out) 662 mem_resp_hit(i) := true.B 663 entries(i).ppn := ptes(index).getPPN() // for last stage 2 translation 664 // when onlystage1, gpf has higher priority 665 entries(i).af := Mux(entries(i).req_info.s2xlate === allStage, false.B, Mux(entries(i).req_info.s2xlate === onlyStage1, ptes(index).isAf() && !ptes(index).isStage1Gpf(io.csr.vsatp.mode), ptes(index).isAf())) 666 entries(i).hptw_resp.gpf := Mux(entries(i).req_info.s2xlate === allStage || entries(i).req_info.s2xlate === onlyStage1, ptes(index).isStage1Gpf(io.csr.vsatp.mode), false.B) 667 } 668 } 669 } 670 671 when (hyper_arb1.io.out.fire) { 672 for (i <- state.indices) { 673 when (state(i) === state_hptw_req && entries(i).ppn === hyper_arb1.io.out.bits.ppn && entries(i).req_info.s2xlate === allStage && hyper_arb1.io.chosen === i.U) { 674 state(i) := state_hptw_resp 675 entries(i).wait_id := hyper_arb1.io.chosen 676 } 677 } 678 } 679 680 when (hyper_arb2.io.out.fire) { 681 for (i <- state.indices) { 682 when (state(i) === state_last_hptw_req && entries(i).ppn === hyper_arb2.io.out.bits.ppn && entries(i).req_info.s2xlate === allStage && hyper_arb2.io.chosen === i.U) { 683 state(i) := state_last_hptw_resp 684 entries(i).wait_id := hyper_arb2.io.chosen 685 } 686 } 687 } 688 689 when (io.hptw.resp.fire) { 690 for (i <- state.indices) { 691 when (state(i) === state_hptw_resp && io.hptw.resp.bits.id === entries(i).wait_id && io.hptw.resp.bits.h_resp.entry.tag === entries(i).ppn) { 692 val check_g_perm_fail = !io.hptw.resp.bits.h_resp.entry.perm.get.r && !(io.csr.priv.mxr && io.hptw.resp.bits.h_resp.entry.perm.get.x) 693 when (check_g_perm_fail || io.hptw.resp.bits.h_resp.gaf || io.hptw.resp.bits.h_resp.gpf) { 694 state(i) := state_mem_out 695 entries(i).hptw_resp := io.hptw.resp.bits.h_resp 696 entries(i).hptw_resp.gpf := io.hptw.resp.bits.h_resp.gpf || check_g_perm_fail 697 entries(i).first_s2xlate_fault := io.hptw.resp.bits.h_resp.gaf || io.hptw.resp.bits.h_resp.gpf 698 }.otherwise{ // change the entry that is waiting hptw resp 699 val need_to_waiting_vec = state.indices.map(i => state(i) === state_mem_waiting && dup(entries(i).req_info.vpn, entries(io.hptw.resp.bits.id).req_info.vpn)) 700 val waiting_index = ParallelMux(need_to_waiting_vec zip entries.map(_.wait_id)) 701 state(i) := Mux(Cat(need_to_waiting_vec).orR, state_mem_waiting, state_addr_check) 702 entries(i).hptw_resp := io.hptw.resp.bits.h_resp 703 entries(i).wait_id := Mux(Cat(need_to_waiting_vec).orR, waiting_index, entries(i).wait_id) 704 //To do: change the entry that is having the same hptw req 705 } 706 } 707 when (state(i) === state_last_hptw_resp && io.hptw.resp.bits.id === entries(i).wait_id && io.hptw.resp.bits.h_resp.entry.tag === entries(i).ppn) { 708 state(i) := state_mem_out 709 entries(i).hptw_resp := io.hptw.resp.bits.h_resp 710 //To do: change the entry that is having the same hptw req 711 } 712 } 713 } 714 when (io.out.fire) { 715 assert(state(mem_ptr) === state_mem_out) 716 state(mem_ptr) := state_idle 717 } 718 mem_resp_hit.map(a => when (a) { a := false.B } ) 719 720 when (io.cache.fire) { 721 state(cache_ptr) := state_idle 722 } 723 XSError(io.out.fire && io.cache.fire && (mem_ptr === cache_ptr), "mem resp and cache fire at the same time at same entry") 724 725 when (flush) { 726 state.map(_ := state_idle) 727 } 728 729 io.in.ready := !full 730 731 io.out.valid := ParallelOR(is_having).asBool 732 io.out.bits.req_info := entries(mem_ptr).req_info 733 io.out.bits.id := mem_ptr 734 io.out.bits.af := entries(mem_ptr).af 735 io.out.bits.h_resp := entries(mem_ptr).hptw_resp 736 io.out.bits.first_s2xlate_fault := entries(mem_ptr).first_s2xlate_fault 737 738 val hptw_req_arb = Module(new Arbiter(new Bundle{ 739 val source = UInt(bSourceWidth.W) 740 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 741 val ppn = UInt(ptePPNLen.W) 742 } , 2)) 743 // first stage 2 translation 744 hptw_req_arb.io.in(0).valid := hyper_arb1.io.out.valid 745 hptw_req_arb.io.in(0).bits.source := hyper_arb1.io.out.bits.req_info.source 746 hptw_req_arb.io.in(0).bits.ppn := hyper_arb1.io.out.bits.ppn 747 hptw_req_arb.io.in(0).bits.id := hyper_arb1.io.chosen 748 hyper_arb1.io.out.ready := hptw_req_arb.io.in(0).ready 749 // last stage 2 translation 750 hptw_req_arb.io.in(1).valid := hyper_arb2.io.out.valid 751 hptw_req_arb.io.in(1).bits.source := hyper_arb2.io.out.bits.req_info.source 752 hptw_req_arb.io.in(1).bits.ppn := hyper_arb2.io.out.bits.ppn 753 hptw_req_arb.io.in(1).bits.id := hyper_arb2.io.chosen 754 hyper_arb2.io.out.ready := hptw_req_arb.io.in(1).ready 755 hptw_req_arb.io.out.ready := io.hptw.req.ready 756 io.hptw.req.valid := hptw_req_arb.io.out.fire && !flush 757 io.hptw.req.bits.gvpn := hptw_req_arb.io.out.bits.ppn 758 io.hptw.req.bits.id := hptw_req_arb.io.out.bits.id 759 io.hptw.req.bits.source := hptw_req_arb.io.out.bits.source 760 761 io.mem.req.valid := mem_arb.io.out.valid && !flush 762 val mem_paddr = MakeAddr(mem_arb.io.out.bits.ppn, getVpnn(mem_arb.io.out.bits.req_info.vpn, 0)) 763 val mem_hpaddr = MakeAddr(mem_arb.io.out.bits.hptw_resp.genPPNS2(get_pn(mem_paddr)), getVpnn(mem_arb.io.out.bits.req_info.vpn, 0)) 764 io.mem.req.bits.addr := Mux(mem_arb.io.out.bits.req_info.s2xlate === allStage, mem_hpaddr, mem_paddr) 765 io.mem.req.bits.id := mem_arb.io.chosen 766 io.mem.req.bits.hptw_bypassed := false.B 767 mem_arb.io.out.ready := io.mem.req.ready 768 val mem_refill_id = RegNext(io.mem.resp.bits.id(log2Up(l2tlbParams.llptwsize)-1, 0)) 769 io.mem.refill := entries(mem_refill_id).req_info 770 io.mem.refill.s2xlate := entries(mem_refill_id).req_info.s2xlate 771 io.mem.buffer_it := mem_resp_hit 772 io.mem.enq_ptr := enq_ptr 773 774 io.cache.valid := Cat(is_cache).orR 775 io.cache.bits := ParallelMux(is_cache, entries.map(_.req_info)) 776 777 XSPerfAccumulate("llptw_in_count", io.in.fire) 778 XSPerfAccumulate("llptw_in_block", io.in.valid && !io.in.ready) 779 for (i <- 0 until 7) { 780 XSPerfAccumulate(s"enq_state${i}", io.in.fire && enq_state === i.U) 781 } 782 for (i <- 0 until (l2tlbParams.llptwsize + 1)) { 783 XSPerfAccumulate(s"util${i}", PopCount(is_emptys.map(!_)) === i.U) 784 XSPerfAccumulate(s"mem_util${i}", PopCount(is_mems) === i.U) 785 XSPerfAccumulate(s"waiting_util${i}", PopCount(is_waiting) === i.U) 786 } 787 XSPerfAccumulate("mem_count", io.mem.req.fire) 788 XSPerfAccumulate("mem_cycle", PopCount(is_waiting) =/= 0.U) 789 XSPerfAccumulate("blocked_in", io.in.valid && !io.in.ready) 790 791 for (i <- 0 until l2tlbParams.llptwsize) { 792 TimeOutAssert(state(i) =/= state_idle, timeOutThreshold, s"missqueue time out no out ${i}") 793 } 794 795 val perfEvents = Seq( 796 ("tlbllptw_incount ", io.in.fire ), 797 ("tlbllptw_inblock ", io.in.valid && !io.in.ready), 798 ("tlbllptw_memcount ", io.mem.req.fire ), 799 ("tlbllptw_memcycle ", PopCount(is_waiting) ), 800 ) 801 generatePerfEvent() 802} 803 804/*========================= HPTW ==============================*/ 805 806/** HPTW : Hypervisor Page Table Walker 807 * the page walker take the virtual machine's page walk. 808 * guest physical address translation, guest physical address -> host physical address 809 **/ 810class HPTWIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 811 val req = Flipped(DecoupledIO(new Bundle { 812 val source = UInt(bSourceWidth.W) 813 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 814 val gvpn = UInt(gvpnLen.W) 815 val ppn = UInt(ppnLen.W) 816 val l3Hit = if (EnableSv48) Some(new Bool()) else None 817 val l2Hit = Bool() 818 val l1Hit = Bool() 819 val bypassed = Bool() // if bypass, don't refill 820 })) 821 val resp = DecoupledIO(new Bundle { 822 val source = UInt(bSourceWidth.W) 823 val resp = Output(new HptwResp()) 824 val id = Output(UInt(bMemID.W)) 825 }) 826 827 val mem = new Bundle { 828 val req = DecoupledIO(new L2TlbMemReqBundle()) 829 val resp = Flipped(ValidIO(UInt(XLEN.W))) 830 val mask = Input(Bool()) 831 } 832 val refill = Output(new Bundle { 833 val req_info = new L2TlbInnerBundle() 834 val level = UInt(log2Up(Level + 1).W) 835 }) 836 val pmp = new Bundle { 837 val req = ValidIO(new PMPReqBundle()) 838 val resp = Flipped(new PMPRespBundle()) 839 } 840} 841 842class HPTW()(implicit p: Parameters) extends XSModule with HasPtwConst { 843 val io = IO(new HPTWIO) 844 val hgatp = io.csr.hgatp 845 val sfence = io.sfence 846 val flush = sfence.valid || hgatp.changed || io.csr.satp.changed || io.csr.vsatp.changed 847 val mode = hgatp.mode 848 849 val level = RegInit(3.U(log2Up(Level + 1).W)) 850 val af_level = RegInit(3.U(log2Up(Level + 1).W)) // access fault return this level 851 val gpaddr = Reg(UInt(GPAddrBits.W)) 852 val req_ppn = Reg(UInt(ppnLen.W)) 853 val vpn = gpaddr(GPAddrBits-1, offLen) 854 val levelNext = level - 1.U 855 val l3Hit = Reg(Bool()) 856 val l2Hit = Reg(Bool()) 857 val l1Hit = Reg(Bool()) 858 val bypassed = Reg(Bool()) 859// val pte = io.mem.resp.bits.MergeRespToPte() 860 val pte = io.mem.resp.bits.asTypeOf(new PteBundle().cloneType) 861 val ppn_l3 = Mux(l3Hit, req_ppn, pte.ppn) 862 val ppn_l2 = Mux(l2Hit, req_ppn, pte.ppn) 863 val ppn_l1 = Mux(l1Hit, req_ppn, pte.ppn) 864 val ppn = Wire(UInt(PAddrBits.W)) 865 val p_pte = MakeAddr(ppn, getVpnn(vpn, level)) 866 val pg_base = Wire(UInt(PAddrBits.W)) 867 val mem_addr = Wire(UInt(PAddrBits.W)) 868 if (EnableSv48) { 869 when (mode === Sv48) { 870 ppn := Mux(af_level === 2.U, ppn_l3, Mux(af_level === 1.U, ppn_l2, ppn_l1)) // for l2, l1 and l3 871 pg_base := MakeGPAddr(hgatp.ppn, getGVpnn(vpn, 3.U, mode = Sv48)) // for l3 872 mem_addr := Mux(af_level === 3.U, pg_base, p_pte) 873 } .otherwise { 874 ppn := Mux(af_level === 1.U, ppn_l2, ppn_l1) //for l1 and l2 875 pg_base := MakeGPAddr(hgatp.ppn, getGVpnn(vpn, 2.U, mode = Sv39)) 876 mem_addr := Mux(af_level === 2.U, pg_base, p_pte) 877 } 878 } else { 879 ppn := Mux(af_level === 1.U, ppn_l2, ppn_l1) //for l1 and l2 880 pg_base := MakeGPAddr(hgatp.ppn, getGVpnn(vpn, 2.U, mode = Sv39)) 881 mem_addr := Mux(af_level === 2.U, pg_base, p_pte) 882 } 883 884 //s/w register 885 val s_pmp_check = RegInit(true.B) 886 val s_mem_req = RegInit(true.B) 887 val w_mem_resp = RegInit(true.B) 888 val idle = RegInit(true.B) 889 val mem_addr_update = RegInit(false.B) 890 val finish = WireInit(false.B) 891 892 val sent_to_pmp = !idle && (!s_pmp_check || mem_addr_update) && !finish 893 val pageFault = pte.isGpf(level) || (!pte.isLeaf() && level === 0.U) 894 val accessFault = RegEnable(io.pmp.resp.ld || io.pmp.resp.mmio, sent_to_pmp) 895 896 val ppn_af = pte.isAf() 897 val find_pte = pte.isLeaf() || ppn_af || pageFault 898 899 val resp_valid = !idle && mem_addr_update && ((w_mem_resp && find_pte) || (s_pmp_check && accessFault)) 900 val id = Reg(UInt(log2Up(l2tlbParams.llptwsize).W)) 901 val source = RegEnable(io.req.bits.source, io.req.fire) 902 903 io.req.ready := idle 904 val resp = Wire(new HptwResp()) 905 resp.apply(pageFault && !accessFault && !ppn_af, accessFault || ppn_af, Mux(accessFault, af_level, level), pte, vpn, hgatp.vmid) 906 io.resp.valid := resp_valid 907 io.resp.bits.id := id 908 io.resp.bits.resp := resp 909 io.resp.bits.source := source 910 911 io.pmp.req.valid := DontCare 912 io.pmp.req.bits.addr := mem_addr 913 io.pmp.req.bits.size := 3.U 914 io.pmp.req.bits.cmd := TlbCmd.read 915 916 io.mem.req.valid := !s_mem_req && !io.mem.mask && !accessFault && s_pmp_check 917 io.mem.req.bits.addr := mem_addr 918 io.mem.req.bits.id := HptwReqId.U(bMemID.W) 919 io.mem.req.bits.hptw_bypassed := bypassed 920 921 io.refill.req_info.vpn := vpn 922 io.refill.level := level 923 io.refill.req_info.source := source 924 io.refill.req_info.s2xlate := onlyStage2 925 when (idle){ 926 when(io.req.fire){ 927 bypassed := io.req.bits.bypassed 928 idle := false.B 929 gpaddr := Cat(io.req.bits.gvpn, 0.U(offLen.W)) 930 accessFault := false.B 931 s_pmp_check := false.B 932 id := io.req.bits.id 933 req_ppn := io.req.bits.ppn 934 if (EnableSv48) { 935 when (mode === Sv48) { 936 level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, Mux(io.req.bits.l3Hit.get, 2.U, 3.U))) 937 af_level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, Mux(io.req.bits.l3Hit.get, 2.U, 3.U))) 938 l3Hit := io.req.bits.l3Hit.get 939 } .otherwise { 940 level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, 2.U)) 941 af_level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, 2.U)) 942 l3Hit := false.B 943 } 944 } else { 945 level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, 2.U)) 946 af_level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, 2.U)) 947 l3Hit := false.B 948 } 949 l2Hit := io.req.bits.l2Hit 950 l1Hit := io.req.bits.l1Hit 951 } 952 } 953 954 when(sent_to_pmp && !mem_addr_update){ 955 s_mem_req := false.B 956 s_pmp_check := true.B 957 } 958 959 when(accessFault && !idle){ 960 s_pmp_check := true.B 961 s_mem_req := true.B 962 w_mem_resp := true.B 963 mem_addr_update := true.B 964 } 965 966 when(io.mem.req.fire){ 967 s_mem_req := true.B 968 w_mem_resp := false.B 969 } 970 971 when(io.mem.resp.fire && !w_mem_resp){ 972 w_mem_resp := true.B 973 af_level := af_level - 1.U 974 mem_addr_update := true.B 975 } 976 977 when(mem_addr_update){ 978 when(!(find_pte || accessFault)){ 979 level := levelNext 980 s_mem_req := false.B 981 mem_addr_update := false.B 982 }.elsewhen(resp_valid){ 983 when(io.resp.fire){ 984 idle := true.B 985 mem_addr_update := false.B 986 accessFault := false.B 987 } 988 finish := true.B 989 } 990 } 991 when (flush) { 992 idle := true.B 993 s_pmp_check := true.B 994 s_mem_req := true.B 995 w_mem_resp := true.B 996 accessFault := false.B 997 mem_addr_update := false.B 998 } 999} 1000