1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import chisel3.internal.naming.chiselName 23import xiangshan._ 24import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 25import utils._ 26import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 27import freechips.rocketchip.tilelink._ 28 29/* ptw cache caches the page table of all the three layers 30 * ptw cache resp at next cycle 31 * the cache should not be blocked 32 * when miss queue if full, just block req outside 33 */ 34 35class PageCachePerPespBundle(implicit p: Parameters) extends PtwBundle { 36 val hit = Bool() 37 val pre = Bool() 38 val ppn = UInt(ppnLen.W) 39 val perm = new PtePermBundle() 40 val ecc = Bool() 41 val level = UInt(2.W) 42 val v = Bool() 43 44 def apply(hit: Bool, pre: Bool, ppn: UInt, perm: PtePermBundle = 0.U.asTypeOf(new PtePermBundle()), 45 ecc: Bool = false.B, level: UInt = 0.U, valid: Bool = true.B) { 46 this.hit := hit && !ecc 47 this.pre := pre 48 this.ppn := ppn 49 this.perm := perm 50 this.ecc := ecc && hit 51 this.level := level 52 this.v := valid 53 } 54} 55 56class PageCacheRespBundle(implicit p: Parameters) extends PtwBundle { 57 val l1 = new PageCachePerPespBundle 58 val l2 = new PageCachePerPespBundle 59 val l3 = new PageCachePerPespBundle 60 val sp = new PageCachePerPespBundle 61} 62 63class PtwCacheReq(implicit p: Parameters) extends PtwBundle { 64 val req_info = new L2TlbInnerBundle() 65 val isFirst = Bool() 66 val bypassed = Vec(3, Bool()) 67} 68 69class PtwCacheIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 70 val req = Flipped(DecoupledIO(new PtwCacheReq())) 71 val resp = DecoupledIO(new Bundle { 72 val req_info = new L2TlbInnerBundle() 73 val isFirst = Bool() 74 val hit = Bool() 75 val prefetch = Bool() // is the entry fetched by prefetch 76 val bypassed = Bool() 77 val toFsm = new Bundle { 78 val l1Hit = Bool() 79 val l2Hit = Bool() 80 val ppn = UInt(ppnLen.W) 81 } 82 val toTlb = new PtwEntry(tagLen = vpnLen, hasPerm = true, hasLevel = true) 83 }) 84 val refill = Flipped(ValidIO(new Bundle { 85 val ptes = UInt(blockBits.W) 86 val req_info = new L2TlbInnerBundle() 87 val level = UInt(log2Up(Level).W) 88 val addr_low = UInt((log2Up(l2tlbParams.blockBytes) - log2Up(XLEN/8)).W) 89 })) 90} 91 92@chiselName 93class PtwCache()(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 94 val io = IO(new PtwCacheIO) 95 96 val ecc = Code.fromString(l2tlbParams.ecc) 97 val l2EntryType = new PTWEntriesWithEcc(ecc, num = PtwL2SectorSize, tagLen = PtwL2TagLen, level = 1, hasPerm = false) 98 val l3EntryType = new PTWEntriesWithEcc(ecc, num = PtwL3SectorSize, tagLen = PtwL3TagLen, level = 2, hasPerm = true) 99 100 // TODO: four caches make the codes dirty, think about how to deal with it 101 102 val sfence = io.sfence 103 val refill = io.refill.bits 104 val refill_prefetch = from_pre(io.refill.bits.req_info.source) 105 val flush = sfence.valid || io.csr.satp.changed 106 107 // when refill, refuce to accept new req 108 val rwHarzad = if (sramSinglePort) io.refill.valid else false.B 109 110 // handle hand signal and req_info 111 // TODO: replace with FlushableQueue 112 val stageReq = Wire(Decoupled(new PtwCacheReq())) // enq stage & read page cache valid 113 val stageDelay = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // page cache resp 114 val stageCheck = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // check hit & check ecc 115 val stageResp = Wire(Decoupled(new PtwCacheReq())) // deq stage 116 stageReq <> io.req 117 PipelineConnect(stageReq, stageDelay(0), stageDelay(1).ready, flush, rwHarzad) 118 InsideStageConnect(stageDelay(0), stageDelay(1), stageReq.fire) 119 PipelineConnect(stageDelay(1), stageCheck(0), stageCheck(1).ready, flush) 120 InsideStageConnect(stageCheck(0), stageCheck(1), stageDelay(1).fire) 121 PipelineConnect(stageCheck(1), stageResp, io.resp.ready, flush) 122 stageResp.ready := !stageResp.valid || io.resp.ready 123 124 // l1: level 0 non-leaf pte 125 val l1 = Reg(Vec(l2tlbParams.l1Size, new PtwEntry(tagLen = PtwL1TagLen))) 126 val l1v = RegInit(0.U(l2tlbParams.l1Size.W)) 127 val l1g = Reg(UInt(l2tlbParams.l1Size.W)) 128 val l1asids = Reg(Vec(l2tlbParams.l1Size, UInt(AsidLength.W))) 129 130 // l2: level 1 non-leaf pte 131 val l2 = Module(new SRAMTemplate( 132 l2EntryType, 133 set = l2tlbParams.l2nSets, 134 way = l2tlbParams.l2nWays, 135 singlePort = sramSinglePort 136 )) 137 val l2v = RegInit(0.U((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 138 val l2g = Reg(UInt((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 139 val l2asids = Reg(Vec(l2tlbParams.l2nSets, Vec(l2tlbParams.l2nWays, UInt(AsidLength.W)))) 140 def getl2vSet(vpn: UInt) = { 141 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 142 val set = genPtwL2SetIdx(vpn) 143 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 144 val l2vVec = l2v.asTypeOf(Vec(l2tlbParams.l2nSets, UInt(l2tlbParams.l2nWays.W))) 145 l2vVec(set) 146 } 147 def getl2asidSet(vpn: UInt) = { 148 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 149 val set = genPtwL2SetIdx(vpn) 150 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 151 l2asids(set) 152 } 153 154 // l3: level 2 leaf pte of 4KB pages 155 val l3 = Module(new SRAMTemplate( 156 l3EntryType, 157 set = l2tlbParams.l3nSets, 158 way = l2tlbParams.l3nWays, 159 singlePort = sramSinglePort 160 )) 161 val l3v = RegInit(0.U((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 162 val l3g = Reg(UInt((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 163 val l3asids = Reg(Vec(l2tlbParams.l3nSets, Vec(l2tlbParams.l3nWays, UInt(AsidLength.W)))) 164 def getl3vSet(vpn: UInt) = { 165 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 166 val set = genPtwL3SetIdx(vpn) 167 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 168 val l3vVec = l3v.asTypeOf(Vec(l2tlbParams.l3nSets, UInt(l2tlbParams.l3nWays.W))) 169 l3vVec(set) 170 } 171 def getl3asidSet(vpn: UInt) = { 172 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 173 val set = genPtwL3SetIdx(vpn) 174 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 175 l3asids(set) 176 } 177 178 // sp: level 0/1 leaf pte of 1GB/2MB super pages 179 val sp = Reg(Vec(l2tlbParams.spSize, new PtwEntry(tagLen = SPTagLen, hasPerm = true, hasLevel = true))) 180 val spv = RegInit(0.U(l2tlbParams.spSize.W)) 181 val spg = Reg(UInt(l2tlbParams.spSize.W)) 182 val spasids = Reg(Vec(l2tlbParams.spSize, UInt(AsidLength.W))) 183 184 // Access Perf 185 val l1AccessPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 186 val l2AccessPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 187 val l3AccessPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 188 val spAccessPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 189 l1AccessPerf.map(_ := false.B) 190 l2AccessPerf.map(_ := false.B) 191 l3AccessPerf.map(_ := false.B) 192 spAccessPerf.map(_ := false.B) 193 194 val stageDelay_valid_1cycle = OneCycleValid(stageReq.fire, flush) // catch ram data 195 val stageCheck_valid_1cycle = OneCycleValid(stageDelay(1).fire, flush) // replace & perf counter 196 val stageResp_valid_1cycle = OneCycleValid(stageCheck(1).fire, flush) // ecc flush 197 198 199 def vpn_match(vpn1: UInt, vpn2: UInt, level: Int) = { 200 vpn1(vpnnLen*3-1, vpnnLen*(2-level)+3) === vpn2(vpnnLen*3-1, vpnnLen*(2-level)+3) 201 } 202 // NOTE: not actually bypassed, just check if hit, re-access the page cache 203 def refill_bypass(vpn: UInt, level: Int) = { 204 io.refill.valid && (level.U === io.refill.bits.level) && vpn_match(io.refill.bits.req_info.vpn, vpn, level), 205 } 206 207 // l1 208 val ptwl1replace = ReplacementPolicy.fromString(l2tlbParams.l1Replacer, l2tlbParams.l1Size) 209 val (l1Hit, l1HitPPN, l1Pre) = { 210 val hitVecT = l1.zipWithIndex.map { case (e, i) => e.hit(stageReq.bits.req_info.vpn, io.csr.satp.asid) && l1v(i) } 211 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 212 213 // stageDelay, but check for l1 214 val hitPPN = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.ppn)), stageDelay_valid_1cycle) 215 val hitPre = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.prefetch)), stageDelay_valid_1cycle) 216 val hit = DataHoldBypass(ParallelOR(hitVec), stageDelay_valid_1cycle) 217 218 when (hit && stageDelay_valid_1cycle) { ptwl1replace.access(OHToUInt(hitVec)) } 219 220 l1AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageDelay_valid_1cycle} 221 for (i <- 0 until l2tlbParams.l1Size) { 222 XSDebug(stageReq.fire, p"[l1] l1(${i.U}) ${l1(i)} hit:${l1(i).hit(stageReq.bits.req_info.vpn, io.csr.satp.asid)}\n") 223 } 224 XSDebug(stageReq.fire, p"[l1] l1v:${Binary(l1v)} hitVecT:${Binary(VecInit(hitVecT).asUInt)}\n") 225 XSDebug(stageDelay(0).valid, p"[l1] l1Hit:${hit} l1HitPPN:0x${Hexadecimal(hitPPN)} hitVec:${VecInit(hitVec).asUInt}\n") 226 227 VecInit(hitVecT).suggestName(s"l1_hitVecT") 228 VecInit(hitVec).suggestName(s"l1_hitVec") 229 230 // synchronize with other entries with RegEnable 231 (RegEnable(hit, stageDelay(1).fire), 232 RegEnable(hitPPN, stageDelay(1).fire), 233 RegEnable(hitPre, stageDelay(1).fire)) 234 } 235 236 // l2 237 val ptwl2replace = ReplacementPolicy.fromString(l2tlbParams.l2Replacer,l2tlbParams.l2nWays,l2tlbParams.l2nSets) 238 val (l2Hit, l2HitPPN, l2Pre, l2eccError) = { 239 val ridx = genPtwL2SetIdx(stageReq.bits.req_info.vpn) 240 l2.io.r.req.valid := stageReq.fire 241 l2.io.r.req.bits.apply(setIdx = ridx) 242 243 // delay one cycle after sram read 244 val data_resp = DataHoldBypass(l2.io.r.resp.data, stageDelay_valid_1cycle) 245 val vVec_delay = DataHoldBypass(getl2vSet(stageDelay(0).bits.req_info.vpn), stageDelay_valid_1cycle) 246 247 // check hit and ecc 248 val check_vpn = stageCheck(0).bits.req_info.vpn 249 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 250 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools() 251 252 val hitVec = VecInit(ramDatas.zip(vVec).map { case (wayData, v) => 253 wayData.entries.hit(check_vpn, io.csr.satp.asid) && v }) 254 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 255 val hitWayData = hitWayEntry.entries 256 val hit = ParallelOR(hitVec) 257 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l2nWays).map(_.U)) 258 val eccError = hitWayEntry.decode() 259 260 ridx.suggestName(s"l2_ridx") 261 ramDatas.suggestName(s"l2_ramDatas") 262 hitVec.suggestName(s"l2_hitVec") 263 hitWayData.suggestName(s"l2_hitWayData") 264 hitWay.suggestName(s"l2_hitWay") 265 266 when (hit && stageCheck_valid_1cycle) { ptwl2replace.access(genPtwL2SetIdx(check_vpn), hitWay) } 267 268 l2AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 269 XSDebug(stageDelay_valid_1cycle, p"[l2] ridx:0x${Hexadecimal(ridx)}\n") 270 for (i <- 0 until l2tlbParams.l2nWays) { 271 XSDebug(stageCheck_valid_1cycle, p"[l2] ramDatas(${i.U}) ${ramDatas(i)} l2v:${vVec(i)} hit:${hit}\n") 272 } 273 XSDebug(stageCheck_valid_1cycle, p"[l2] l2Hit:${hit} l2HitPPN:0x${Hexadecimal(hitWayData.ppns(genPtwL2SectorIdx(check_vpn)))} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${vVec}\n") 274 275 (hit, hitWayData.ppns(genPtwL2SectorIdx(check_vpn)), hitWayData.prefetch, eccError) 276 } 277 278 // l3 279 val ptwl3replace = ReplacementPolicy.fromString(l2tlbParams.l3Replacer,l2tlbParams.l3nWays,l2tlbParams.l3nSets) 280 val (l3Hit, l3HitData, l3Pre, l3eccError) = { 281 val ridx = genPtwL3SetIdx(stageReq.bits.req_info.vpn) 282 l3.io.r.req.valid := stageReq.fire 283 l3.io.r.req.bits.apply(setIdx = ridx) 284 285 // delay one cycle after sram read 286 val data_resp = DataHoldBypass(l3.io.r.resp.data, stageDelay_valid_1cycle) 287 val vVec_delay = DataHoldBypass(getl3vSet(stageDelay(0).bits.req_info.vpn), stageDelay_valid_1cycle) 288 val bypass_delay = DataHoldBypass(refill_bypass(stageDelay(0).bits.req_info.vpn, 2), stageDelay_valid_1cycle || io.refill.valid) 289 290 // check hit and ecc 291 val check_vpn = stageCheck(0).bits.req_info.vpn 292 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 293 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools() 294 295 val hitVec = VecInit(ramDatas.zip(vVec).map{ case (wayData, v) => 296 wayData.entries.hit(check_vpn, io.csr.satp.asid) && v }) 297 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 298 val hitWayData = hitWayEntry.entries 299 val hitWayEcc = hitWayEntry.ecc 300 val hit = ParallelOR(hitVec) 301 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l3nWays).map(_.U)) 302 val eccError = hitWayEntry.decode() 303 304 when (hit && stageCheck_valid_1cycle) { ptwl3replace.access(genPtwL3SetIdx(check_vpn), hitWay) } 305 306 l3AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 307 XSDebug(stageReq.fire, p"[l3] ridx:0x${Hexadecimal(ridx)}\n") 308 for (i <- 0 until l2tlbParams.l3nWays) { 309 XSDebug(stageCheck_valid_1cycle, p"[l3] ramDatas(${i.U}) ${ramDatas(i)} l3v:${vVec(i)} hit:${hitVec(i)}\n") 310 } 311 XSDebug(stageCheck_valid_1cycle, p"[l3] l3Hit:${hit} l3HitData:${hitWayData} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} v:${vVec}\n") 312 313 ridx.suggestName(s"l3_ridx") 314 ramDatas.suggestName(s"l3_ramDatas") 315 hitVec.suggestName(s"l3_hitVec") 316 hitWay.suggestName(s"l3_hitWay") 317 318 (hit, hitWayData, hitWayData.prefetch, eccError) 319 } 320 val l3HitPPN = l3HitData.ppns(genPtwL3SectorIdx(stageCheck(0).bits.req_info.vpn)) 321 val l3HitPerm = l3HitData.perms.getOrElse(0.U.asTypeOf(Vec(PtwL3SectorSize, new PtePermBundle)))(genPtwL3SectorIdx(stageCheck(0).bits.req_info.vpn)) 322 val l3HitValid = l3HitData.vs(genPtwL3SectorIdx(stageCheck(0).bits.req_info.vpn)) 323 324 // super page 325 val spreplace = ReplacementPolicy.fromString(l2tlbParams.spReplacer, l2tlbParams.spSize) 326 val (spHit, spHitData, spPre, spValid) = { 327 val hitVecT = sp.zipWithIndex.map { case (e, i) => e.hit(stageReq.bits.req_info.vpn, io.csr.satp.asid) && spv(i) } 328 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 329 val hitData = ParallelPriorityMux(hitVec zip sp) 330 val hit = ParallelOR(hitVec) 331 332 when (hit && stageDelay_valid_1cycle) { spreplace.access(OHToUInt(hitVec)) } 333 334 spAccessPerf.zip(hitVec).map{ case (s, h) => s := h && stageDelay_valid_1cycle } 335 for (i <- 0 until l2tlbParams.spSize) { 336 XSDebug(stageReq.fire, p"[sp] sp(${i.U}) ${sp(i)} hit:${sp(i).hit(stageReq.bits.req_info.vpn, io.csr.satp.asid)} spv:${spv(i)}\n") 337 } 338 XSDebug(stageDelay_valid_1cycle, p"[sp] spHit:${hit} spHitData:${hitData} hitVec:${Binary(VecInit(hitVec).asUInt)}\n") 339 340 VecInit(hitVecT).suggestName(s"sp_hitVecT") 341 VecInit(hitVec).suggestName(s"sp_hitVec") 342 343 (RegEnable(hit, stageDelay(1).fire), 344 RegEnable(hitData, stageDelay(1).fire), 345 RegEnable(hitData.prefetch, stageDelay(1).fire), 346 RegEnable(hitData.v, stageDelay(1).fire())) 347 } 348 val spHitPerm = spHitData.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)) 349 val spHitLevel = spHitData.level.getOrElse(0.U) 350 351 val check_res = Wire(new PageCacheRespBundle) 352 check_res.l1.apply(l1Hit, l1Pre, l1HitPPN) 353 check_res.l2.apply(l2Hit, l2Pre, l2HitPPN, ecc = l2eccError) 354 check_res.l3.apply(l3Hit, l3Pre, l3HitPPN, l3HitPerm, l3eccError, valid = l3HitValid) 355 check_res.sp.apply(spHit, spPre, spHitData.ppn, spHitPerm, false.B, spHitLevel, spValid) 356 357 val resp_res = Reg(new PageCacheRespBundle) 358 when (stageCheck(1).fire) { resp_res := check_res } 359 360 // stageResp bypass 361 val bypassed = Wire(Vec(3, Bool())) 362 bypassed.indices.foreach(i => 363 bypassed(i) := stageResp.bits.bypassed(i) || 364 ValidHoldBypass(refill_bypass(stageResp.bits.req_info.vpn, i), 365 OneCycleValid(stageCheck(1).fire, false.B) || io.refill.valid) 366 ) 367 368 io.resp.bits.req_info := stageResp.bits.req_info 369 io.resp.bits.isFirst := stageResp.bits.isFirst 370 io.resp.bits.hit := resp_res.l3.hit || resp_res.sp.hit 371 io.resp.bits.bypassed := bypassed(2) || (bypassed(1) && !resp_res.l2.hit) || (bypassed(0) && !resp_res.l1.hit) 372 io.resp.bits.prefetch := resp_res.l3.pre && resp_res.l3.hit || resp_res.sp.pre && resp_res.sp.hit 373 io.resp.bits.toFsm.l1Hit := resp_res.l1.hit 374 io.resp.bits.toFsm.l2Hit := resp_res.l2.hit 375 io.resp.bits.toFsm.ppn := Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l1.ppn) 376 io.resp.bits.toTlb.tag := stageResp.bits.req_info.vpn 377 io.resp.bits.toTlb.asid := io.csr.satp.asid // DontCare 378 io.resp.bits.toTlb.ppn := Mux(resp_res.l3.hit, resp_res.l3.ppn, resp_res.sp.ppn) 379 io.resp.bits.toTlb.perm.map(_ := Mux(resp_res.l3.hit, resp_res.l3.perm, resp_res.sp.perm)) 380 io.resp.bits.toTlb.level.map(_ := Mux(resp_res.l3.hit, 2.U, resp_res.sp.level)) 381 io.resp.bits.toTlb.prefetch := from_pre(stageResp.bits.req_info.source) 382 io.resp.bits.toTlb.v := Mux(resp_res.sp.hit, resp_res.sp.v, resp_res.l3.v) 383 io.resp.valid := stageResp.valid 384 XSError(stageResp.valid && resp_res.l3.hit && resp_res.sp.hit, "normal page and super page both hit") 385 XSError(stageResp.valid && io.resp.bits.hit && bypassed(2), "page cache, bypassed but hit") 386 387 // refill Perf 388 val l1RefillPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 389 val l2RefillPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 390 val l3RefillPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 391 val spRefillPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 392 l1RefillPerf.map(_ := false.B) 393 l2RefillPerf.map(_ := false.B) 394 l3RefillPerf.map(_ := false.B) 395 spRefillPerf.map(_ := false.B) 396 397 // refill 398 l2.io.w.req <> DontCare 399 l3.io.w.req <> DontCare 400 l2.io.w.req.valid := false.B 401 l3.io.w.req.valid := false.B 402 403 def get_part(data: UInt, index: UInt): UInt = { 404 val inner_data = data.asTypeOf(Vec(data.getWidth / XLEN, UInt(XLEN.W))) 405 inner_data(index) 406 } 407 408 val memRdata = refill.ptes 409 val memSelData = get_part(memRdata, refill.addr_low) 410 val memPtes = (0 until (l2tlbParams.blockBytes/(XLEN/8))).map(i => memRdata((i+1)*XLEN-1, i*XLEN).asTypeOf(new PteBundle)) 411 val memPte = memSelData.asTypeOf(new PteBundle) 412 413 memPte.suggestName("memPte") 414 415 // TODO: handle sfenceLatch outsize 416 when (io.refill.valid && !memPte.isPf(refill.level) && !flush ) { 417 when (refill.level === 0.U && !memPte.isLeaf()) { 418 // val refillIdx = LFSR64()(log2Up(l2tlbParams.l1Size)-1,0) // TODO: may be LRU 419 val refillIdx = replaceWrapper(l1v, ptwl1replace.way) 420 refillIdx.suggestName(s"PtwL1RefillIdx") 421 val rfOH = UIntToOH(refillIdx) 422 l1(refillIdx).refill( 423 refill.req_info.vpn, 424 io.csr.satp.asid, 425 memSelData, 426 0.U, 427 refill_prefetch 428 ) 429 ptwl1replace.access(refillIdx) 430 l1v := l1v | rfOH 431 l1g := (l1g & ~rfOH) | Mux(memPte.perm.g, rfOH, 0.U) 432 433 for (i <- 0 until l2tlbParams.l1Size) { 434 l1RefillPerf(i) := i.U === refillIdx 435 } 436 437 XSDebug(p"[l1 refill] refillIdx:${refillIdx} refillEntry:${l1(refillIdx).genPtwEntry(refill.req_info.vpn, io.csr.satp.asid, memSelData, 0.U, prefetch = refill_prefetch)}\n") 438 XSDebug(p"[l1 refill] l1v:${Binary(l1v)}->${Binary(l1v | rfOH)} l1g:${Binary(l1g)}->${Binary((l1g & ~rfOH) | Mux(memPte.perm.g, rfOH, 0.U))}\n") 439 440 refillIdx.suggestName(s"l1_refillIdx") 441 rfOH.suggestName(s"l1_rfOH") 442 } 443 444 when (refill.level === 1.U && !memPte.isLeaf()) { 445 val refillIdx = genPtwL2SetIdx(refill.req_info.vpn) 446 val victimWay = replaceWrapper(getl2vSet(refill.req_info.vpn), ptwl2replace.way(refillIdx)) 447 val victimWayOH = UIntToOH(victimWay) 448 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 449 val wdata = Wire(l2EntryType) 450 wdata.gen( 451 vpn = refill.req_info.vpn, 452 asid = io.csr.satp.asid, 453 data = memRdata, 454 levelUInt = 1.U, 455 refill_prefetch 456 ) 457 l2.io.w.apply( 458 valid = true.B, 459 setIdx = refillIdx, 460 data = wdata, 461 waymask = victimWayOH 462 ) 463 ptwl2replace.access(refillIdx, victimWay) 464 l2v := l2v | rfvOH 465 l2g := l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 466 467 for (i <- 0 until l2tlbParams.l2nWays) { 468 l2RefillPerf(i) := i.U === victimWay 469 } 470 471 XSDebug(p"[l2 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 472 XSDebug(p"[l2 refill] refilldata:0x${wdata}\n") 473 XSDebug(p"[l2 refill] l2v:${Binary(l2v)} -> ${Binary(l2v | rfvOH)}\n") 474 XSDebug(p"[l2 refill] l2g:${Binary(l2g)} -> ${Binary(l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 475 476 refillIdx.suggestName(s"l2_refillIdx") 477 victimWay.suggestName(s"l2_victimWay") 478 victimWayOH.suggestName(s"l2_victimWayOH") 479 rfvOH.suggestName(s"l2_rfvOH") 480 } 481 482 when (refill.level === 2.U && memPte.isLeaf()) { 483 val refillIdx = genPtwL3SetIdx(refill.req_info.vpn) 484 val victimWay = replaceWrapper(getl3vSet(refill.req_info.vpn), ptwl3replace.way(refillIdx)) 485 val victimWayOH = UIntToOH(victimWay) 486 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 487 val wdata = Wire(l3EntryType) 488 wdata.gen( 489 vpn = refill.req_info.vpn, 490 asid = io.csr.satp.asid, 491 data = memRdata, 492 levelUInt = 2.U, 493 refill_prefetch 494 ) 495 l3.io.w.apply( 496 valid = true.B, 497 setIdx = refillIdx, 498 data = wdata, 499 waymask = victimWayOH 500 ) 501 ptwl3replace.access(refillIdx, victimWay) 502 l3v := l3v | rfvOH 503 l3g := l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 504 505 for (i <- 0 until l2tlbParams.l3nWays) { 506 l3RefillPerf(i) := i.U === victimWay 507 } 508 509 XSDebug(p"[l3 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 510 XSDebug(p"[l3 refill] refilldata:0x${wdata}\n") 511 XSDebug(p"[l3 refill] l3v:${Binary(l3v)} -> ${Binary(l3v | rfvOH)}\n") 512 XSDebug(p"[l3 refill] l3g:${Binary(l3g)} -> ${Binary(l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 513 514 refillIdx.suggestName(s"l3_refillIdx") 515 victimWay.suggestName(s"l3_victimWay") 516 victimWayOH.suggestName(s"l3_victimWayOH") 517 rfvOH.suggestName(s"l3_rfvOH") 518 } 519 } 520 521 // misc entries: super & invalid 522 when (io.refill.valid && !flush && (refill.level === 0.U || refill.level === 1.U) && (memPte.isLeaf() || memPte.isPf(refill.level))) { 523 val refillIdx = spreplace.way// LFSR64()(log2Up(l2tlbParams.spSize)-1,0) // TODO: may be LRU 524 val rfOH = UIntToOH(refillIdx) 525 sp(refillIdx).refill( 526 refill.req_info.vpn, 527 io.csr.satp.asid, 528 memSelData, 529 refill.level, 530 refill_prefetch, 531 !memPte.isPf(refill.level), 532 ) 533 spreplace.access(refillIdx) 534 spv := spv | rfOH 535 spg := spg & ~rfOH | Mux(memPte.perm.g, rfOH, 0.U) 536 537 for (i <- 0 until l2tlbParams.spSize) { 538 spRefillPerf(i) := i.U === refillIdx 539 } 540 541 XSDebug(p"[sp refill] refillIdx:${refillIdx} refillEntry:${sp(refillIdx).genPtwEntry(refill.req_info.vpn, io.csr.satp.asid, memSelData, refill.level, refill_prefetch)}\n") 542 XSDebug(p"[sp refill] spv:${Binary(spv)}->${Binary(spv | rfOH)} spg:${Binary(spg)}->${Binary(spg & ~rfOH | Mux(memPte.perm.g, rfOH, 0.U))}\n") 543 544 refillIdx.suggestName(s"sp_refillIdx") 545 rfOH.suggestName(s"sp_rfOH") 546 } 547 548 val l2eccFlush = resp_res.l2.ecc && stageResp_valid_1cycle // RegNext(l2eccError, init = false.B) 549 val l3eccFlush = resp_res.l3.ecc && stageResp_valid_1cycle // RegNext(l3eccError, init = false.B) 550 val eccVpn = stageResp.bits.req_info.vpn 551 552 XSError(l2eccFlush, "l2tlb.cache.l2 ecc error. Should not happen at sim stage") 553 XSError(l3eccFlush, "l2tlb.cache.l3 ecc error. Should not happen at sim stage") 554 when (l2eccFlush) { 555 val flushSetIdxOH = UIntToOH(genPtwL2SetIdx(eccVpn)) 556 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l2nWays, a.asUInt) }).asUInt 557 l2v := l2v & ~flushMask 558 l2g := l2g & ~flushMask 559 } 560 561 when (l3eccFlush) { 562 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(eccVpn)) 563 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 564 l3v := l3v & ~flushMask 565 l3g := l3g & ~flushMask 566 } 567 568 // sfence 569 when (sfence.valid) { 570 val l1asidhit = VecInit(l1asids.map(_ === sfence.bits.asid)).asUInt 571 val spasidhit = VecInit(spasids.map(_ === sfence.bits.asid)).asUInt 572 val sfence_vpn = sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen) 573 574 when (sfence.bits.rs1/*va*/) { 575 when (sfence.bits.rs2) { 576 // all va && all asid 577 l1v := 0.U 578 l2v := 0.U 579 l3v := 0.U 580 spv := 0.U 581 } .otherwise { 582 // all va && specific asid except global 583 584 l1v := l1v & (~l1asidhit | l1g) 585 l2v := l2v & l2g 586 l3v := l3v & l3g 587 spv := spv & (~spasidhit | spg) 588 } 589 } .otherwise { 590 // val flushMask = UIntToOH(genTlbL2Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen))) 591 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(sfence_vpn)) 592 // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l3nWays, _.asUInt))).asUInt 593 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 594 flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH") 595 flushMask.suggestName(s"sfence_nrs1_flushMask") 596 597 when (sfence.bits.rs2) { 598 // specific leaf of addr && all asid 599 l3v := l3v & ~flushMask 600 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence.bits.asid, ignoreAsid = true))).asUInt | spg) 601 } .otherwise { 602 // specific leaf of addr && specific asid 603 l3v := l3v & (~flushMask | l3g) 604 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence.bits.asid))).asUInt | spg) 605 } 606 } 607 } 608 609 def InsideStageConnect(in: DecoupledIO[PtwCacheReq], out: DecoupledIO[PtwCacheReq], InFire: Bool): Unit = { 610 in.ready := !in.valid || out.ready 611 out.valid := in.valid 612 out.bits := in.bits 613 out.bits.bypassed.zip(in.bits.bypassed).zipWithIndex.map{ case (b, i) => 614 b._1 := b._2 || DataHoldBypass(refill_bypass(in.bits.req_info.vpn, i), OneCycleValid(InFire, false.B) || io.refill.valid) 615 } 616 } 617 618 // Perf Count 619 val resp_l3 = resp_res.l3.hit 620 val resp_sp = resp_res.sp.hit 621 val resp_l1_pre = resp_res.l1.pre 622 val resp_l2_pre = resp_res.l2.pre 623 val resp_l3_pre = resp_res.l3.pre 624 val resp_sp_pre = resp_res.sp.pre 625 val base_valid_access_0 = !from_pre(io.resp.bits.req_info.source) && io.resp.fire() 626 XSPerfAccumulate("access", base_valid_access_0) 627 XSPerfAccumulate("l1_hit", base_valid_access_0 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 628 XSPerfAccumulate("l2_hit", base_valid_access_0 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 629 XSPerfAccumulate("l3_hit", base_valid_access_0 && resp_l3) 630 XSPerfAccumulate("sp_hit", base_valid_access_0 && resp_sp) 631 XSPerfAccumulate("pte_hit",base_valid_access_0 && io.resp.bits.hit) 632 633 XSPerfAccumulate("l1_hit_pre", base_valid_access_0 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 634 XSPerfAccumulate("l2_hit_pre", base_valid_access_0 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 635 XSPerfAccumulate("l3_hit_pre", base_valid_access_0 && resp_l3_pre && resp_l3) 636 XSPerfAccumulate("sp_hit_pre", base_valid_access_0 && resp_sp_pre && resp_sp) 637 XSPerfAccumulate("pte_hit_pre",base_valid_access_0 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 638 639 val base_valid_access_1 = from_pre(io.resp.bits.req_info.source) && io.resp.fire() 640 XSPerfAccumulate("pre_access", base_valid_access_1) 641 XSPerfAccumulate("pre_l1_hit", base_valid_access_1 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 642 XSPerfAccumulate("pre_l2_hit", base_valid_access_1 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 643 XSPerfAccumulate("pre_l3_hit", base_valid_access_1 && resp_l3) 644 XSPerfAccumulate("pre_sp_hit", base_valid_access_1 && resp_sp) 645 XSPerfAccumulate("pre_pte_hit",base_valid_access_1 && io.resp.bits.hit) 646 647 XSPerfAccumulate("pre_l1_hit_pre", base_valid_access_1 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 648 XSPerfAccumulate("pre_l2_hit_pre", base_valid_access_1 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 649 XSPerfAccumulate("pre_l3_hit_pre", base_valid_access_1 && resp_l3_pre && resp_l3) 650 XSPerfAccumulate("pre_sp_hit_pre", base_valid_access_1 && resp_sp_pre && resp_sp) 651 XSPerfAccumulate("pre_pte_hit_pre",base_valid_access_1 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 652 653 val base_valid_access_2 = stageResp.bits.isFirst && !from_pre(io.resp.bits.req_info.source) && io.resp.fire() 654 XSPerfAccumulate("access_first", base_valid_access_2) 655 XSPerfAccumulate("l1_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 656 XSPerfAccumulate("l2_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 657 XSPerfAccumulate("l3_hit_first", base_valid_access_2 && resp_l3) 658 XSPerfAccumulate("sp_hit_first", base_valid_access_2 && resp_sp) 659 XSPerfAccumulate("pte_hit_first",base_valid_access_2 && io.resp.bits.hit) 660 661 XSPerfAccumulate("l1_hit_pre_first", base_valid_access_2 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 662 XSPerfAccumulate("l2_hit_pre_first", base_valid_access_2 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 663 XSPerfAccumulate("l3_hit_pre_first", base_valid_access_2 && resp_l3_pre && resp_l3) 664 XSPerfAccumulate("sp_hit_pre_first", base_valid_access_2 && resp_sp_pre && resp_sp) 665 XSPerfAccumulate("pte_hit_pre_first",base_valid_access_2 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 666 667 val base_valid_access_3 = stageResp.bits.isFirst && from_pre(io.resp.bits.req_info.source) && io.resp.fire() 668 XSPerfAccumulate("pre_access_first", base_valid_access_3) 669 XSPerfAccumulate("pre_l1_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 670 XSPerfAccumulate("pre_l2_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 671 XSPerfAccumulate("pre_l3_hit_first", base_valid_access_3 && resp_l3) 672 XSPerfAccumulate("pre_sp_hit_first", base_valid_access_3 && resp_sp) 673 XSPerfAccumulate("pre_pte_hit_first", base_valid_access_3 && io.resp.bits.hit) 674 675 XSPerfAccumulate("pre_l1_hit_pre_first", base_valid_access_3 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 676 XSPerfAccumulate("pre_l2_hit_pre_first", base_valid_access_3 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 677 XSPerfAccumulate("pre_l3_hit_pre_first", base_valid_access_3 && resp_l3_pre && resp_l3) 678 XSPerfAccumulate("pre_sp_hit_pre_first", base_valid_access_3 && resp_sp_pre && resp_sp) 679 XSPerfAccumulate("pre_pte_hit_pre_first",base_valid_access_3 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 680 681 XSPerfAccumulate("rwHarzad", io.req.valid && !io.req.ready) 682 XSPerfAccumulate("out_blocked", io.resp.valid && !io.resp.ready) 683 l1AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1AccessIndex${i}", l) } 684 l2AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2AccessIndex${i}", l) } 685 l3AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3AccessIndex${i}", l) } 686 spAccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPAccessIndex${i}", l) } 687 l1RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1RefillIndex${i}", l) } 688 l2RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2RefillIndex${i}", l) } 689 l3RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3RefillIndex${i}", l) } 690 spRefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPRefillIndex${i}", l) } 691 692 XSPerfAccumulate("l1Refill", Cat(l1RefillPerf).orR) 693 XSPerfAccumulate("l2Refill", Cat(l2RefillPerf).orR) 694 XSPerfAccumulate("l3Refill", Cat(l3RefillPerf).orR) 695 XSPerfAccumulate("spRefill", Cat(spRefillPerf).orR) 696 XSPerfAccumulate("l1Refill_pre", Cat(l1RefillPerf).orR && refill_prefetch) 697 XSPerfAccumulate("l2Refill_pre", Cat(l2RefillPerf).orR && refill_prefetch) 698 XSPerfAccumulate("l3Refill_pre", Cat(l3RefillPerf).orR && refill_prefetch) 699 XSPerfAccumulate("spRefill_pre", Cat(spRefillPerf).orR && refill_prefetch) 700 701 // debug 702 XSDebug(sfence.valid, p"[sfence] original v and g vector:\n") 703 XSDebug(sfence.valid, p"[sfence] l1v:${Binary(l1v)}\n") 704 XSDebug(sfence.valid, p"[sfence] l2v:${Binary(l2v)}\n") 705 XSDebug(sfence.valid, p"[sfence] l3v:${Binary(l3v)}\n") 706 XSDebug(sfence.valid, p"[sfence] l3g:${Binary(l3g)}\n") 707 XSDebug(sfence.valid, p"[sfence] spv:${Binary(spv)}\n") 708 XSDebug(RegNext(sfence.valid), p"[sfence] new v and g vector:\n") 709 XSDebug(RegNext(sfence.valid), p"[sfence] l1v:${Binary(l1v)}\n") 710 XSDebug(RegNext(sfence.valid), p"[sfence] l2v:${Binary(l2v)}\n") 711 XSDebug(RegNext(sfence.valid), p"[sfence] l3v:${Binary(l3v)}\n") 712 XSDebug(RegNext(sfence.valid), p"[sfence] l3g:${Binary(l3g)}\n") 713 XSDebug(RegNext(sfence.valid), p"[sfence] spv:${Binary(spv)}\n") 714 715 val perfEvents = Seq( 716 ("access ", base_valid_access_0 ), 717 ("l1_hit ", l1Hit ), 718 ("l2_hit ", l2Hit ), 719 ("l3_hit ", l3Hit ), 720 ("sp_hit ", spHit ), 721 ("pte_hit ", l3Hit || spHit ), 722 ("rwHarzad ", io.req.valid && !io.req.ready ), 723 ("out_blocked ", io.resp.valid && !io.resp.ready), 724 ) 725 generatePerfEvent() 726} 727