1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import chisel3.internal.naming.chiselName 23import xiangshan._ 24import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 25import utils._ 26import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 27import freechips.rocketchip.tilelink._ 28 29/* ptw cache caches the page table of all the three layers 30 * ptw cache resp at next cycle 31 * the cache should not be blocked 32 * when miss queue if full, just block req outside 33 */ 34class PtwCacheIO()(implicit p: Parameters) extends PtwBundle { 35 val req = Flipped(DecoupledIO(new L2TlbInnerBundle())) 36 val req_isFirst = Input(Bool()) // only for perf counter 37 val resp = DecoupledIO(new Bundle { 38 val req_info = new L2TlbInnerBundle() 39 val hit = Bool() 40 val prefetch = Bool() // is the entry fetched by prefetch 41 val toFsm = new Bundle { 42 val l1Hit = Bool() 43 val l2Hit = Bool() 44 val ppn = UInt(ppnLen.W) 45 } 46 val toTlb = new PtwEntry(tagLen = vpnLen, hasPerm = true, hasLevel = true) 47 }) 48 val refill = Flipped(ValidIO(new Bundle { 49 val ptes = UInt(blockBits.W) 50 val req_info = new L2TlbInnerBundle() 51 val level = UInt(log2Up(Level).W) 52 val addr_low = UInt((log2Up(l2tlbParams.blockBytes) - log2Up(XLEN/8)).W) 53 })) 54 val sfence = Input(new SfenceBundle) 55 val csr = Input(new TlbCsrBundle) 56} 57 58@chiselName 59class PtwCache()(implicit p: Parameters) extends XSModule with HasPtwConst { 60 val io = IO(new PtwCacheIO) 61 62 val ecc = Code.fromString(l2tlbParams.ecc) 63 val l2EntryType = new PTWEntriesWithEcc(ecc, num = PtwL2SectorSize, tagLen = PtwL2TagLen, level = 1, hasPerm = false) 64 val l3EntryType = new PTWEntriesWithEcc(ecc, num = PtwL3SectorSize, tagLen = PtwL3TagLen, level = 2, hasPerm = true) 65 66 // TODO: four caches make the codes dirty, think about how to deal with it 67 68 val sfence = io.sfence 69 val refill = io.refill.bits 70 val refill_prefetch = from_pre(io.refill.bits.req_info.source) 71 72 val first_valid = io.req.valid 73 val first_fire = first_valid && io.req.ready 74 val first_req = io.req.bits 75 val second_ready = Wire(Bool()) 76 val second_valid = ValidHold(first_fire && !sfence.valid, io.resp.fire(), sfence.valid) 77 val second_req = RegEnable(first_req, first_fire) 78 // NOTE: if ptw cache resp may be blocked, hard to handle refill 79 // when miss queue is full, please to block itlb and dtlb input 80 val second_isFirst = RegEnable(io.req_isFirst, first_fire) // only for perf counter 81 82 // when refill, refuce to accept new req 83 val rwHarzad = if (sramSinglePort) io.refill.valid else false.B 84 io.req.ready := !rwHarzad && second_ready 85 // NOTE: when write, don't ready, whe 86 // when replay, just come in, out make sure resp.fire() 87 88 // l1: level 0 non-leaf pte 89 val l1 = Reg(Vec(l2tlbParams.l1Size, new PtwEntry(tagLen = PtwL1TagLen))) 90 val l1v = RegInit(0.U(l2tlbParams.l1Size.W)) 91 val l1g = Reg(UInt(l2tlbParams.l1Size.W)) 92 val l1asids = Reg(Vec(l2tlbParams.l1Size, UInt(AsidLength.W))) 93 94 // l2: level 1 non-leaf pte 95 val l2 = Module(new SRAMTemplate( 96 l2EntryType, 97 set = l2tlbParams.l2nSets, 98 way = l2tlbParams.l2nWays, 99 singlePort = sramSinglePort 100 )) 101 val l2v = RegInit(0.U((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 102 val l2g = Reg(UInt((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 103 val l2asids = Reg(Vec(l2tlbParams.l2nSets, Vec(l2tlbParams.l2nWays, UInt(AsidLength.W)))) 104 def getl2vSet(vpn: UInt) = { 105 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 106 val set = genPtwL2SetIdx(vpn) 107 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 108 val l2vVec = l2v.asTypeOf(Vec(l2tlbParams.l2nSets, UInt(l2tlbParams.l2nWays.W))) 109 l2vVec(set) 110 } 111 def getl2asidSet(vpn: UInt) = { 112 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 113 val set = genPtwL2SetIdx(vpn) 114 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 115 l2asids(set) 116 } 117 118 // l3: level 2 leaf pte of 4KB pages 119 val l3 = Module(new SRAMTemplate( 120 l3EntryType, 121 set = l2tlbParams.l3nSets, 122 way = l2tlbParams.l3nWays, 123 singlePort = sramSinglePort 124 )) 125 val l3v = RegInit(0.U((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 126 val l3g = Reg(UInt((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 127 val l3asids = Reg(Vec(l2tlbParams.l3nSets, Vec(l2tlbParams.l3nWays, UInt(AsidLength.W)))) 128 def getl3vSet(vpn: UInt) = { 129 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 130 val set = genPtwL3SetIdx(vpn) 131 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 132 val l3vVec = l3v.asTypeOf(Vec(l2tlbParams.l3nSets, UInt(l2tlbParams.l3nWays.W))) 133 l3vVec(set) 134 } 135 def getl3asidSet(vpn: UInt) = { 136 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 137 val set = genPtwL3SetIdx(vpn) 138 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 139 l3asids(set) 140 } 141 142 // sp: level 0/1 leaf pte of 1GB/2MB super pages 143 val sp = Reg(Vec(l2tlbParams.spSize, new PtwEntry(tagLen = SPTagLen, hasPerm = true, hasLevel = true))) 144 val spv = RegInit(0.U(l2tlbParams.spSize.W)) 145 val spg = Reg(UInt(l2tlbParams.spSize.W)) 146 val spasids = Reg(Vec(l2tlbParams.spSize, UInt(AsidLength.W))) 147 148 // Access Perf 149 val l1AccessPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 150 val l2AccessPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 151 val l3AccessPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 152 val spAccessPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 153 l1AccessPerf.map(_ := false.B) 154 l2AccessPerf.map(_ := false.B) 155 l3AccessPerf.map(_ := false.B) 156 spAccessPerf.map(_ := false.B) 157 158 val cache_read_valid = OneCycleValid(first_fire, sfence.valid) 159 // l1 160 val ptwl1replace = ReplacementPolicy.fromString(l2tlbParams.l1Replacer, l2tlbParams.l1Size) 161 val (l1Hit, l1HitPPN, l1Pre) = { 162 val hitVecT = l1.zipWithIndex.map { case (e, i) => e.hit(first_req.vpn, io.csr.satp.asid) && l1v(i) } 163 val hitVec = hitVecT.map(RegEnable(_, first_fire)) 164 val hitPPN = ParallelPriorityMux(hitVec zip l1.map(_.ppn)) 165 val hitPre = ParallelPriorityMux(hitVec zip l1.map(_.prefetch)) 166 val hit = ParallelOR(hitVec) && cache_read_valid 167 168 when (hit) { ptwl1replace.access(OHToUInt(hitVec)) } 169 170 l1AccessPerf.zip(hitVec).map{ case (l, h) => l := h && RegNext(first_fire)} 171 for (i <- 0 until l2tlbParams.l1Size) { 172 XSDebug(first_fire, p"[l1] l1(${i.U}) ${l1(i)} hit:${l1(i).hit(first_req.vpn, io.csr.satp.asid)}\n") 173 } 174 XSDebug(first_fire, p"[l1] l1v:${Binary(l1v)} hitVecT:${Binary(VecInit(hitVecT).asUInt)}\n") 175 XSDebug(second_valid, p"[l1] l1Hit:${hit} l1HitPPN:0x${Hexadecimal(hitPPN)} hitVec:${VecInit(hitVec).asUInt}\n") 176 177 VecInit(hitVecT).suggestName(s"l1_hitVecT") 178 VecInit(hitVec).suggestName(s"l1_hitVec") 179 180 (hit, hitPPN, hitPre) 181 } 182 183 // l2 184 val ptwl2replace = ReplacementPolicy.fromString(l2tlbParams.l2Replacer,l2tlbParams.l2nWays,l2tlbParams.l2nSets) 185 val (l2Hit, l2HitPPN, l2Pre, l2eccError) = { 186 val ridx = genPtwL2SetIdx(first_req.vpn) 187 val vidx = RegEnable(VecInit(getl2vSet(first_req.vpn).asBools), first_fire) 188 val asids_idx = RegEnable(getl2asidSet(first_req.vpn), first_fire) 189 l2.io.r.req.valid := first_fire 190 l2.io.r.req.bits.apply(setIdx = ridx) 191 val ramDatas = l2.io.r.resp.data 192 // val hitVec = VecInit(ramDatas.map{wayData => wayData.hit(first_req.vpn) }) 193 val hitVec = VecInit(ramDatas.zip(vidx).map { case (wayData, v) => wayData.entries.hit(second_req.vpn, io.csr.satp.asid) && v }) 194 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 195 val hitWayData = hitWayEntry.entries 196 val hitWayEcc = hitWayEntry.ecc 197 val hit = ParallelOR(hitVec) && cache_read_valid && RegNext(l2.io.r.req.ready, init = false.B) 198 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l2nWays).map(_.U)) 199 200 val eccError = ecc.decode(Cat(hitWayEcc, hitWayData.asUInt())).error 201 202 ridx.suggestName(s"l2_ridx") 203 vidx.suggestName(s"l2_vidx") 204 ramDatas.suggestName(s"l2_ramDatas") 205 hitVec.suggestName(s"l2_hitVec") 206 hitWayData.suggestName(s"l2_hitWayData") 207 hitWay.suggestName(s"l2_hitWay") 208 209 when (hit) { ptwl2replace.access(genPtwL2SetIdx(second_req.vpn), hitWay) } 210 211 l2AccessPerf.zip(hitVec).map{ case (l, h) => l := h && RegNext(first_fire) } 212 XSDebug(first_fire, p"[l2] ridx:0x${Hexadecimal(ridx)}\n") 213 for (i <- 0 until l2tlbParams.l2nWays) { 214 XSDebug(RegNext(first_fire), p"[l2] ramDatas(${i.U}) ${ramDatas(i)} l2v:${vidx(i)} hit:${ramDatas(i).entries.hit(second_req.vpn, io.csr.satp.asid)}\n") 215 } 216 XSDebug(second_valid, p"[l2] l2Hit:${hit} l2HitPPN:0x${Hexadecimal(hitWayData.ppns(genPtwL2SectorIdx(second_req.vpn)))} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${Binary(vidx.asUInt)}\n") 217 218 (hit && !eccError, hitWayData.ppns(genPtwL2SectorIdx(second_req.vpn)), hitWayData.prefetch, hit && eccError) 219 } 220 221 // l3 222 val ptwl3replace = ReplacementPolicy.fromString(l2tlbParams.l3Replacer,l2tlbParams.l3nWays,l2tlbParams.l3nSets) 223 val (l3Hit, l3HitData, l3Pre, l3eccError) = { 224 val ridx = genPtwL3SetIdx(first_req.vpn) 225 val vidx = RegEnable(VecInit(getl3vSet(first_req.vpn).asBools), first_fire) 226 val asids_idx = RegEnable(getl3asidSet(first_req.vpn), first_fire) 227 l3.io.r.req.valid := first_fire 228 l3.io.r.req.bits.apply(setIdx = ridx) 229 val ramDatas = l3.io.r.resp.data 230 val hitVec = VecInit(ramDatas.zip(vidx).map{ case (wayData, v) => wayData.entries.hit(second_req.vpn, io.csr.satp.asid) && v }) 231 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 232 val hitWayData = hitWayEntry.entries 233 val hitWayEcc = hitWayEntry.ecc 234 val hit = ParallelOR(hitVec) && cache_read_valid && RegNext(l3.io.r.req.ready, init = false.B) 235 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l3nWays).map(_.U)) 236 237 val eccError = ecc.decode(Cat(hitWayEcc, hitWayData.asUInt())).error 238 239 when (hit) { ptwl3replace.access(genPtwL3SetIdx(second_req.vpn), hitWay) } 240 241 l3AccessPerf.zip(hitVec).map{ case (l, h) => l := h && RegNext(first_fire) } 242 XSDebug(first_fire, p"[l3] ridx:0x${Hexadecimal(ridx)}\n") 243 for (i <- 0 until l2tlbParams.l3nWays) { 244 XSDebug(RegNext(first_fire), p"[l3] ramDatas(${i.U}) ${ramDatas(i)} l3v:${vidx(i)} hit:${ramDatas(i).entries.hit(second_req.vpn, io.csr.satp.asid)}\n") 245 } 246 XSDebug(second_valid, p"[l3] l3Hit:${hit} l3HitData:${hitWayData} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${Binary(vidx.asUInt)}\n") 247 248 ridx.suggestName(s"l3_ridx") 249 vidx.suggestName(s"l3_vidx") 250 ramDatas.suggestName(s"l3_ramDatas") 251 hitVec.suggestName(s"l3_hitVec") 252 hitWay.suggestName(s"l3_hitWay") 253 254 (hit && !eccError, hitWayData, hitWayData.prefetch, hit && eccError) 255 } 256 val l3HitPPN = l3HitData.ppns(genPtwL3SectorIdx(second_req.vpn)) 257 val l3HitPerm = l3HitData.perms.getOrElse(0.U.asTypeOf(Vec(PtwL3SectorSize, new PtePermBundle)))(genPtwL3SectorIdx(second_req.vpn)) 258 259 // super page 260 val spreplace = ReplacementPolicy.fromString(l2tlbParams.spReplacer, l2tlbParams.spSize) 261 val (spHit, spHitData, spPre) = { 262 val hitVecT = sp.zipWithIndex.map { case (e, i) => e.hit(first_req.vpn, io.csr.satp.asid) && spv(i) } 263 val hitVec = hitVecT.map(RegEnable(_, first_fire)) 264 val hitData = ParallelPriorityMux(hitVec zip sp) 265 val hit = ParallelOR(hitVec) && cache_read_valid 266 267 when (hit) { spreplace.access(OHToUInt(hitVec)) } 268 269 spAccessPerf.zip(hitVec).map{ case (s, h) => s := h && RegNext(first_fire) } 270 for (i <- 0 until l2tlbParams.spSize) { 271 XSDebug(first_fire, p"[sp] sp(${i.U}) ${sp(i)} hit:${sp(i).hit(first_req.vpn, io.csr.satp.asid)} spv:${spv(i)}\n") 272 } 273 XSDebug(second_valid, p"[sp] spHit:${hit} spHitData:${hitData} hitVec:${Binary(VecInit(hitVec).asUInt)}\n") 274 275 VecInit(hitVecT).suggestName(s"sp_hitVecT") 276 VecInit(hitVec).suggestName(s"sp_hitVec") 277 278 (hit, hitData, hitData.prefetch) 279 } 280 val spHitPerm = spHitData.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)) 281 val spHitLevel = spHitData.level.getOrElse(0.U) 282 283 val resp = Wire(io.resp.bits.cloneType) 284 val resp_latch = RegEnable(resp, io.resp.valid && !io.resp.ready) 285 val resp_latch_valid = ValidHold(io.resp.valid && !io.resp.ready, io.resp.fire(), sfence.valid) 286 second_ready := !second_valid || io.resp.fire() 287 resp.req_info := second_req 288 resp.hit := l3Hit || spHit 289 resp.prefetch := l3Pre && l3Hit || spPre && spHit 290 resp.toFsm.l1Hit := l1Hit 291 resp.toFsm.l2Hit := l2Hit 292 resp.toFsm.ppn := Mux(l2Hit, l2HitPPN, l1HitPPN) 293 resp.toTlb.tag := second_req.vpn 294 resp.toTlb.asid := io.csr.satp.asid // DontCare 295 resp.toTlb.ppn := Mux(l3Hit, l3HitPPN, spHitData.ppn) 296 resp.toTlb.perm.map(_ := Mux(l3Hit, l3HitPerm, spHitPerm)) 297 resp.toTlb.level.map(_ := Mux(l3Hit, 2.U, spHitLevel)) 298 resp.toTlb.prefetch := from_pre(second_req.source) 299 300 io.resp.valid := second_valid 301 io.resp.bits := Mux(resp_latch_valid, resp_latch, resp) 302 assert(!(l3Hit && spHit), "normal page and super page both hit") 303 304 // refill Perf 305 val l1RefillPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 306 val l2RefillPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 307 val l3RefillPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 308 val spRefillPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 309 l1RefillPerf.map(_ := false.B) 310 l2RefillPerf.map(_ := false.B) 311 l3RefillPerf.map(_ := false.B) 312 spRefillPerf.map(_ := false.B) 313 314 // refill 315 l2.io.w.req <> DontCare 316 l3.io.w.req <> DontCare 317 l2.io.w.req.valid := false.B 318 l3.io.w.req.valid := false.B 319 320 def get_part(data: UInt, index: UInt): UInt = { 321 val inner_data = data.asTypeOf(Vec(data.getWidth / XLEN, UInt(XLEN.W))) 322 inner_data(index) 323 } 324 325 val memRdata = refill.ptes 326 val memSelData = get_part(memRdata, refill.addr_low) 327 val memPtes = (0 until (l2tlbParams.blockBytes/(XLEN/8))).map(i => memRdata((i+1)*XLEN-1, i*XLEN).asTypeOf(new PteBundle)) 328 val memPte = memSelData.asTypeOf(new PteBundle) 329 330 memPte.suggestName("memPte") 331 332 // TODO: handle sfenceLatch outsize 333 when (io.refill.valid && !memPte.isPf(refill.level) && !sfence.valid ) { 334 when (refill.level === 0.U && !memPte.isLeaf()) { 335 // val refillIdx = LFSR64()(log2Up(l2tlbParams.l1Size)-1,0) // TODO: may be LRU 336 val refillIdx = replaceWrapper(l1v, ptwl1replace.way) 337 refillIdx.suggestName(s"PtwL1RefillIdx") 338 val rfOH = UIntToOH(refillIdx) 339 l1(refillIdx).refill( 340 refill.req_info.vpn, 341 io.csr.satp.asid, 342 memSelData, 343 0.U, 344 refill_prefetch 345 ) 346 ptwl1replace.access(refillIdx) 347 l1v := l1v | rfOH 348 l1g := (l1g & ~rfOH) | Mux(memPte.perm.g, rfOH, 0.U) 349 350 for (i <- 0 until l2tlbParams.l1Size) { 351 l1RefillPerf(i) := i.U === refillIdx 352 } 353 354 XSDebug(p"[l1 refill] refillIdx:${refillIdx} refillEntry:${l1(refillIdx).genPtwEntry(refill.req_info.vpn, io.csr.satp.asid, memSelData, 0.U, refill_prefetch)}\n") 355 XSDebug(p"[l1 refill] l1v:${Binary(l1v)}->${Binary(l1v | rfOH)} l1g:${Binary(l1g)}->${Binary((l1g & ~rfOH) | Mux(memPte.perm.g, rfOH, 0.U))}\n") 356 357 refillIdx.suggestName(s"l1_refillIdx") 358 rfOH.suggestName(s"l1_rfOH") 359 } 360 361 when (refill.level === 1.U && !memPte.isLeaf()) { 362 val refillIdx = genPtwL2SetIdx(refill.req_info.vpn) 363 val victimWay = replaceWrapper(RegEnable(VecInit(getl2vSet(refill.req_info.vpn).asBools).asUInt, first_fire), ptwl2replace.way(refillIdx)) 364 val victimWayOH = UIntToOH(victimWay) 365 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 366 val wdata = Wire(l2EntryType) 367 wdata.entries := wdata.entries.genEntries( 368 vpn = refill.req_info.vpn, 369 asid = io.csr.satp.asid, 370 data = memRdata, 371 levelUInt = 1.U, 372 refill_prefetch 373 ) 374 wdata.ecc := ecc.encode(wdata.entries.asUInt()) >> wdata.entries.getWidth 375 l2.io.w.apply( 376 valid = true.B, 377 setIdx = refillIdx, 378 data = wdata, 379 waymask = victimWayOH 380 ) 381 ptwl2replace.access(refillIdx, victimWay) 382 l2v := l2v | rfvOH 383 l2g := l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 384 385 for (i <- 0 until l2tlbParams.l2nWays) { 386 l2RefillPerf(i) := i.U === victimWay 387 } 388 389 XSDebug(p"[l2 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 390 XSDebug(p"[l2 refill] refilldata:0x${wdata}\n") 391 XSDebug(p"[l2 refill] l2v:${Binary(l2v)} -> ${Binary(l2v | rfvOH)}\n") 392 XSDebug(p"[l2 refill] l2g:${Binary(l2g)} -> ${Binary(l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 393 394 refillIdx.suggestName(s"l2_refillIdx") 395 victimWay.suggestName(s"l2_victimWay") 396 victimWayOH.suggestName(s"l2_victimWayOH") 397 rfvOH.suggestName(s"l2_rfvOH") 398 } 399 400 when (refill.level === 2.U && memPte.isLeaf()) { 401 val refillIdx = genPtwL3SetIdx(refill.req_info.vpn) 402 val victimWay = replaceWrapper(RegEnable(VecInit(getl3vSet(refill.req_info.vpn).asBools).asUInt, first_fire), ptwl3replace.way(refillIdx)) 403 val victimWayOH = UIntToOH(victimWay) 404 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 405 val wdata = Wire(l3EntryType) 406 wdata.entries := wdata.entries.genEntries( 407 vpn = refill.req_info.vpn, 408 asid = io.csr.satp.asid, 409 data = memRdata, 410 levelUInt = 2.U, 411 refill_prefetch 412 ) 413 wdata.ecc := ecc.encode(wdata.entries.asUInt()) >> wdata.entries.getWidth 414 l3.io.w.apply( 415 valid = true.B, 416 setIdx = refillIdx, 417 data = wdata, 418 waymask = victimWayOH 419 ) 420 ptwl3replace.access(refillIdx, victimWay) 421 l3v := l3v | rfvOH 422 l3g := l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 423 424 for (i <- 0 until l2tlbParams.l3nWays) { 425 l3RefillPerf(i) := i.U === victimWay 426 } 427 428 XSDebug(p"[l3 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 429 XSDebug(p"[l3 refill] refilldata:0x${wdata}\n") 430 XSDebug(p"[l3 refill] l3v:${Binary(l3v)} -> ${Binary(l3v | rfvOH)}\n") 431 XSDebug(p"[l3 refill] l3g:${Binary(l3g)} -> ${Binary(l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 432 433 refillIdx.suggestName(s"l3_refillIdx") 434 victimWay.suggestName(s"l3_victimWay") 435 victimWayOH.suggestName(s"l3_victimWayOH") 436 rfvOH.suggestName(s"l3_rfvOH") 437 } 438 when ((refill.level === 0.U || refill.level === 1.U) && memPte.isLeaf()) { 439 val refillIdx = spreplace.way// LFSR64()(log2Up(l2tlbParams.spSize)-1,0) // TODO: may be LRU 440 val rfOH = UIntToOH(refillIdx) 441 sp(refillIdx).refill( 442 refill.req_info.vpn, 443 io.csr.satp.asid, 444 memSelData, 445 refill.level, 446 refill_prefetch 447 ) 448 spreplace.access(refillIdx) 449 spv := spv | rfOH 450 spg := spg & ~rfOH | Mux(memPte.perm.g, rfOH, 0.U) 451 452 for (i <- 0 until l2tlbParams.spSize) { 453 spRefillPerf(i) := i.U === refillIdx 454 } 455 456 XSDebug(p"[sp refill] refillIdx:${refillIdx} refillEntry:${sp(refillIdx).genPtwEntry(refill.req_info.vpn, io.csr.satp.asid, memSelData, refill.level, refill_prefetch)}\n") 457 XSDebug(p"[sp refill] spv:${Binary(spv)}->${Binary(spv | rfOH)} spg:${Binary(spg)}->${Binary(spg & ~rfOH | Mux(memPte.perm.g, rfOH, 0.U))}\n") 458 459 refillIdx.suggestName(s"sp_refillIdx") 460 rfOH.suggestName(s"sp_rfOH") 461 } 462 } 463 464 val l2eccFlush = RegNext(l2eccError, init = false.B) 465 val l3eccFlush = RegNext(l3eccError, init = false.B) 466 val eccVpn = RegNext(second_req.vpn) 467 468 assert(!l2eccFlush) 469 assert(!l3eccFlush) 470 when (l2eccFlush) { 471 val flushSetIdxOH = UIntToOH(genPtwL2SetIdx(eccVpn)) 472 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l2nWays, a.asUInt) }).asUInt 473 l2v := l2v & ~flushMask 474 l2g := l2g & ~flushMask 475 } 476 477 when (l3eccFlush) { 478 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(eccVpn)) 479 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 480 l3v := l3v & ~flushMask 481 l3g := l3g & ~flushMask 482 } 483 484 // sfence 485 when (sfence.valid) { 486 val l1asidhit = VecInit(l1asids.map(_ === sfence.bits.asid)).asUInt 487 val spasidhit = VecInit(spasids.map(_ === sfence.bits.asid)).asUInt 488 val sfence_vpn = sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen) 489 490 when (sfence.bits.rs1/*va*/) { 491 when (sfence.bits.rs2) { 492 // all va && all asid 493 l1v := 0.U 494 l2v := 0.U 495 l3v := 0.U 496 spv := 0.U 497 } .otherwise { 498 // all va && specific asid except global 499 500 l1v := l1v & (~l1asidhit | l1g) 501 l2v := l2v & l2g 502 l3v := l3v & l3g 503 spv := spv & (~spasidhit | spg) 504 } 505 } .otherwise { 506 // val flushMask = UIntToOH(genTlbL2Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen))) 507 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(sfence_vpn)) 508 // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l3nWays, _.asUInt))).asUInt 509 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 510 flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH") 511 flushMask.suggestName(s"sfence_nrs1_flushMask") 512 513 when (sfence.bits.rs2) { 514 // specific leaf of addr && all asid 515 l3v := l3v & ~flushMask 516 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence.bits.asid, ignoreAsid = true))).asUInt | spg) 517 } .otherwise { 518 // specific leaf of addr && specific asid 519 l3v := l3v & (~flushMask | l3g) 520 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence.bits.asid))).asUInt | spg) 521 } 522 } 523 } 524 525 // Perf Count 526 val resp_l3 = DataHoldBypass(l3Hit, io.resp.valid && !resp_latch_valid).asBool() 527 val resp_sp = DataHoldBypass(spHit, io.resp.valid && !resp_latch_valid).asBool() 528 val resp_l1_pre = DataHoldBypass(l1Pre, io.resp.valid && !resp_latch_valid).asBool() 529 val resp_l2_pre = DataHoldBypass(l2Pre, io.resp.valid && !resp_latch_valid).asBool() 530 val resp_l3_pre = DataHoldBypass(l3Pre, io.resp.valid && !resp_latch_valid).asBool() 531 val resp_sp_pre = DataHoldBypass(spPre, io.resp.valid && !resp_latch_valid).asBool() 532 val base_valid_access_0 = !from_pre(io.resp.bits.req_info.source) && io.resp.fire() 533 XSPerfAccumulate("access", base_valid_access_0) 534 XSPerfAccumulate("l1_hit", base_valid_access_0 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 535 XSPerfAccumulate("l2_hit", base_valid_access_0 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 536 XSPerfAccumulate("l3_hit", base_valid_access_0 && resp_l3) 537 XSPerfAccumulate("sp_hit", base_valid_access_0 && resp_sp) 538 XSPerfAccumulate("pte_hit",base_valid_access_0 && io.resp.bits.hit) 539 540 XSPerfAccumulate("l1_hit_pre", base_valid_access_0 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 541 XSPerfAccumulate("l2_hit_pre", base_valid_access_0 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 542 XSPerfAccumulate("l3_hit_pre", base_valid_access_0 && resp_l3_pre && resp_l3) 543 XSPerfAccumulate("sp_hit_pre", base_valid_access_0 && resp_sp_pre && resp_sp) 544 XSPerfAccumulate("pte_hit_pre",base_valid_access_0 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 545 546 val base_valid_access_1 = from_pre(io.resp.bits.req_info.source) && io.resp.fire() 547 XSPerfAccumulate("pre_access", base_valid_access_1) 548 XSPerfAccumulate("pre_l1_hit", base_valid_access_1 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 549 XSPerfAccumulate("pre_l2_hit", base_valid_access_1 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 550 XSPerfAccumulate("pre_l3_hit", base_valid_access_1 && resp_l3) 551 XSPerfAccumulate("pre_sp_hit", base_valid_access_1 && resp_sp) 552 XSPerfAccumulate("pre_pte_hit",base_valid_access_1 && io.resp.bits.hit) 553 554 XSPerfAccumulate("pre_l1_hit_pre", base_valid_access_1 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 555 XSPerfAccumulate("pre_l2_hit_pre", base_valid_access_1 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 556 XSPerfAccumulate("pre_l3_hit_pre", base_valid_access_1 && resp_l3_pre && resp_l3) 557 XSPerfAccumulate("pre_sp_hit_pre", base_valid_access_1 && resp_sp_pre && resp_sp) 558 XSPerfAccumulate("pre_pte_hit_pre",base_valid_access_1 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 559 560 val base_valid_access_2 = second_isFirst && !from_pre(io.resp.bits.req_info.source) && io.resp.fire() 561 XSPerfAccumulate("access_first", base_valid_access_2) 562 XSPerfAccumulate("l1_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 563 XSPerfAccumulate("l2_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 564 XSPerfAccumulate("l3_hit_first", base_valid_access_2 && resp_l3) 565 XSPerfAccumulate("sp_hit_first", base_valid_access_2 && resp_sp) 566 XSPerfAccumulate("pte_hit_first",base_valid_access_2 && io.resp.bits.hit) 567 568 XSPerfAccumulate("l1_hit_pre_first", base_valid_access_2 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 569 XSPerfAccumulate("l2_hit_pre_first", base_valid_access_2 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 570 XSPerfAccumulate("l3_hit_pre_first", base_valid_access_2 && resp_l3_pre && resp_l3) 571 XSPerfAccumulate("sp_hit_pre_first", base_valid_access_2 && resp_sp_pre && resp_sp) 572 XSPerfAccumulate("pte_hit_pre_first",base_valid_access_2 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 573 574 val base_valid_access_3 = second_isFirst && from_pre(io.resp.bits.req_info.source) && io.resp.fire() 575 XSPerfAccumulate("pre_access_first", base_valid_access_3) 576 XSPerfAccumulate("pre_l1_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 577 XSPerfAccumulate("pre_l2_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 578 XSPerfAccumulate("pre_l3_hit_first", base_valid_access_3 && resp_l3) 579 XSPerfAccumulate("pre_sp_hit_first", base_valid_access_3 && resp_sp) 580 XSPerfAccumulate("pre_pte_hit_first", base_valid_access_3 && io.resp.bits.hit) 581 582 XSPerfAccumulate("pre_l1_hit_pre_first", base_valid_access_3 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 583 XSPerfAccumulate("pre_l2_hit_pre_first", base_valid_access_3 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 584 XSPerfAccumulate("pre_l3_hit_pre_first", base_valid_access_3 && resp_l3_pre && resp_l3) 585 XSPerfAccumulate("pre_sp_hit_pre_first", base_valid_access_3 && resp_sp_pre && resp_sp) 586 XSPerfAccumulate("pre_pte_hit_pre_first",base_valid_access_3 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 587 588 XSPerfAccumulate("rwHarzad", io.req.valid && !io.req.ready) 589 XSPerfAccumulate("out_blocked", io.resp.valid && !io.resp.ready) 590 l1AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1AccessIndex${i}", l) } 591 l2AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2AccessIndex${i}", l) } 592 l3AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3AccessIndex${i}", l) } 593 spAccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPAccessIndex${i}", l) } 594 l1RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1RefillIndex${i}", l) } 595 l2RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2RefillIndex${i}", l) } 596 l3RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3RefillIndex${i}", l) } 597 spRefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPRefillIndex${i}", l) } 598 599 XSPerfAccumulate("l1Refill", Cat(l1RefillPerf).orR) 600 XSPerfAccumulate("l2Refill", Cat(l2RefillPerf).orR) 601 XSPerfAccumulate("l3Refill", Cat(l3RefillPerf).orR) 602 XSPerfAccumulate("spRefill", Cat(spRefillPerf).orR) 603 XSPerfAccumulate("l1Refill_pre", Cat(l1RefillPerf).orR && refill_prefetch) 604 XSPerfAccumulate("l2Refill_pre", Cat(l2RefillPerf).orR && refill_prefetch) 605 XSPerfAccumulate("l3Refill_pre", Cat(l3RefillPerf).orR && refill_prefetch) 606 XSPerfAccumulate("spRefill_pre", Cat(spRefillPerf).orR && refill_prefetch) 607 608 // debug 609 XSDebug(sfence.valid, p"[sfence] original v and g vector:\n") 610 XSDebug(sfence.valid, p"[sfence] l1v:${Binary(l1v)}\n") 611 XSDebug(sfence.valid, p"[sfence] l2v:${Binary(l2v)}\n") 612 XSDebug(sfence.valid, p"[sfence] l3v:${Binary(l3v)}\n") 613 XSDebug(sfence.valid, p"[sfence] l3g:${Binary(l3g)}\n") 614 XSDebug(sfence.valid, p"[sfence] spv:${Binary(spv)}\n") 615 XSDebug(RegNext(sfence.valid), p"[sfence] new v and g vector:\n") 616 XSDebug(RegNext(sfence.valid), p"[sfence] l1v:${Binary(l1v)}\n") 617 XSDebug(RegNext(sfence.valid), p"[sfence] l2v:${Binary(l2v)}\n") 618 XSDebug(RegNext(sfence.valid), p"[sfence] l3v:${Binary(l3v)}\n") 619 XSDebug(RegNext(sfence.valid), p"[sfence] l3g:${Binary(l3g)}\n") 620 XSDebug(RegNext(sfence.valid), p"[sfence] spv:${Binary(spv)}\n") 621} 622