1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import xiangshan._ 23import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 24import utils._ 25import utility._ 26import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 27import freechips.rocketchip.tilelink._ 28 29/* ptw cache caches the page table of all the three layers 30 * ptw cache resp at next cycle 31 * the cache should not be blocked 32 * when miss queue if full, just block req outside 33 */ 34 35class PageCachePerPespBundle(implicit p: Parameters) extends PtwBundle { 36 val hit = Bool() 37 val pre = Bool() 38 val ppn = if (HasHExtension) UInt((vpnLen max ppnLen).W) else UInt(ppnLen.W) 39 val perm = new PtePermBundle() 40 val ecc = Bool() 41 val level = UInt(2.W) 42 val v = Bool() 43 44 def apply(hit: Bool, pre: Bool, ppn: UInt, perm: PtePermBundle = 0.U.asTypeOf(new PtePermBundle()), 45 ecc: Bool = false.B, level: UInt = 0.U, valid: Bool = true.B) { 46 this.hit := hit && !ecc 47 this.pre := pre 48 this.ppn := ppn 49 this.perm := perm 50 this.ecc := ecc && hit 51 this.level := level 52 this.v := valid 53 } 54} 55 56class PageCacheMergePespBundle(implicit p: Parameters) extends PtwBundle { 57 assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 58 val hit = Bool() 59 val pre = Bool() 60 val ppn = Vec(tlbcontiguous, if(HasHExtension) UInt((vpnLen max ppnLen).W) else UInt(ppnLen.W)) 61 val perm = Vec(tlbcontiguous, new PtePermBundle()) 62 val ecc = Bool() 63 val level = UInt(2.W) 64 val v = Vec(tlbcontiguous, Bool()) 65 66 def apply(hit: Bool, pre: Bool, ppn: Vec[UInt], perm: Vec[PtePermBundle] = Vec(tlbcontiguous, 0.U.asTypeOf(new PtePermBundle())), 67 ecc: Bool = false.B, level: UInt = 0.U, valid: Vec[Bool] = Vec(tlbcontiguous, true.B)) { 68 this.hit := hit && !ecc 69 this.pre := pre 70 this.ppn := ppn 71 this.perm := perm 72 this.ecc := ecc && hit 73 this.level := level 74 this.v := valid 75 } 76} 77 78class PageCacheRespBundle(implicit p: Parameters) extends PtwBundle { 79 val l1 = new PageCachePerPespBundle 80 val l2 = new PageCachePerPespBundle 81 val l3 = new PageCacheMergePespBundle 82 val sp = new PageCachePerPespBundle 83} 84 85class PtwCacheReq(implicit p: Parameters) extends PtwBundle { 86 val req_info = new L2TlbInnerBundle() 87 val isFirst = Bool() 88 val bypassed = Vec(3, Bool()) 89 val isHptw = Bool() 90 val hptwId = UInt(log2Up(l2tlbParams.llptwsize).W) 91} 92 93class PtwCacheIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 94 val req = Flipped(DecoupledIO(new PtwCacheReq())) 95 val resp = DecoupledIO(new Bundle { 96 val req_info = new L2TlbInnerBundle() 97 val isFirst = Bool() 98 val hit = Bool() 99 val prefetch = Bool() // is the entry fetched by prefetch 100 val bypassed = Bool() 101 val toFsm = new Bundle { 102 val l1Hit = Bool() 103 val l2Hit = Bool() 104 val ppn = if(HasHExtension) UInt((vpnLen.max(ppnLen)).W) else UInt(ppnLen.W) 105 val stage1Hit = Bool() // find stage 1 pte in cache, but need to search stage 2 pte in cache at PTW 106 } 107 val toTlb = new PtwMergeResp() 108 val isHptw = Bool() 109 val toHptw = new Bundle { 110 val l1Hit = Bool() 111 val l2Hit = Bool() 112 val ppn = UInt(ppnLen.W) 113 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 114 val resp = new HptwResp() // used if hit 115 } 116 }) 117 val refill = Flipped(ValidIO(new Bundle { 118 val ptes = UInt(blockBits.W) 119 val levelOH = new Bundle { 120 // NOTE: levelOH has (Level+1) bits, each stands for page cache entries 121 val sp = Bool() 122 val l3 = Bool() 123 val l2 = Bool() 124 val l1 = Bool() 125 def apply(levelUInt: UInt, valid: Bool) = { 126 sp := RegNext((levelUInt === 0.U || levelUInt === 1.U) && valid, false.B) 127 l3 := RegNext((levelUInt === 2.U) & valid, false.B) 128 l2 := RegNext((levelUInt === 1.U) & valid, false.B) 129 l1 := RegNext((levelUInt === 0.U) & valid, false.B) 130 } 131 } 132 // duplicate level and sel_pte for each page caches, for better fanout 133 val req_info_dup = Vec(3, new L2TlbInnerBundle()) 134 val level_dup = Vec(3, UInt(log2Up(Level).W)) 135 val sel_pte_dup = Vec(3, UInt(XLEN.W)) 136 })) 137 val sfence_dup = Vec(4, Input(new SfenceBundle())) 138 val csr_dup = Vec(3, Input(new TlbCsrBundle())) 139} 140 141class PtwCache()(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 142 val io = IO(new PtwCacheIO) 143 val ecc = Code.fromString(l2tlbParams.ecc) 144 val l2EntryType = new PTWEntriesWithEcc(ecc, num = PtwL2SectorSize, tagLen = PtwL2TagLen, level = 1, hasPerm = false) 145 val l3EntryType = new PTWEntriesWithEcc(ecc, num = PtwL3SectorSize, tagLen = PtwL3TagLen, level = 2, hasPerm = true) 146 147 // TODO: four caches make the codes dirty, think about how to deal with it 148 149 val sfence_dup = io.sfence_dup 150 val refill = io.refill.bits 151 val refill_prefetch_dup = io.refill.bits.req_info_dup.map(a => from_pre(a.source)) 152 val flush_dup = sfence_dup.zip(io.csr_dup).map(f => f._1.valid || f._2.satp.changed || f._2.vsatp.changed || f._2.hgatp.changed) 153 val flush = flush_dup(0) 154 155 // when refill, refuce to accept new req 156 val rwHarzad = if (sramSinglePort) io.refill.valid else false.B 157 158 // handle hand signal and req_info 159 // TODO: replace with FlushableQueue 160 val stageReq = Wire(Decoupled(new PtwCacheReq())) // enq stage & read page cache valid 161 val stageDelay = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // page cache resp 162 val stageCheck = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // check hit & check ecc 163 val stageResp = Wire(Decoupled(new PtwCacheReq())) // deq stage 164 165 val stageDelay_valid_1cycle = OneCycleValid(stageReq.fire, flush) // catch ram data 166 val stageCheck_valid_1cycle = OneCycleValid(stageDelay(1).fire, flush) // replace & perf counter 167 val stageResp_valid_1cycle_dup = Wire(Vec(2, Bool())) 168 stageResp_valid_1cycle_dup.map(_ := OneCycleValid(stageCheck(1).fire, flush)) // ecc flush 169 170 stageReq <> io.req 171 PipelineConnect(stageReq, stageDelay(0), stageDelay(1).ready, flush, rwHarzad) 172 InsideStageConnect(stageDelay(0), stageDelay(1), stageDelay_valid_1cycle) 173 PipelineConnect(stageDelay(1), stageCheck(0), stageCheck(1).ready, flush) 174 InsideStageConnect(stageCheck(0), stageCheck(1), stageCheck_valid_1cycle) 175 PipelineConnect(stageCheck(1), stageResp, io.resp.ready, flush) 176 stageResp.ready := !stageResp.valid || io.resp.ready 177 178 // l1: level 0 non-leaf pte 179 val l1 = Reg(Vec(l2tlbParams.l1Size, new PtwEntry(tagLen = PtwL1TagLen))) 180 val l1v = RegInit(0.U(l2tlbParams.l1Size.W)) 181 val l1g = Reg(UInt(l2tlbParams.l1Size.W)) 182 val l1asids = l1.map(_.asid) 183 val l1vmids = l1.map(_.vmid) 184 val l1h = Reg(Vec(l2tlbParams.l1Size, UInt(2.W))) // 0 bit: s2xlate, 1 bit: stage 1 or stage 2 185 186 // l2: level 1 non-leaf pte 187 val l2 = Module(new SRAMTemplate( 188 l2EntryType, 189 set = l2tlbParams.l2nSets, 190 way = l2tlbParams.l2nWays, 191 singlePort = sramSinglePort 192 )) 193 val l2v = RegInit(0.U((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 194 val l2g = Reg(UInt((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 195 val l2h = Reg(Vec(l2tlbParams.l2nSets, Vec(l2tlbParams.l2nWays, UInt(2.W)))) 196 def getl2vSet(vpn: UInt) = { 197 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 198 val set = genPtwL2SetIdx(vpn) 199 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 200 val l2vVec = l2v.asTypeOf(Vec(l2tlbParams.l2nSets, UInt(l2tlbParams.l2nWays.W))) 201 l2vVec(set) 202 } 203 def getl2hSet(vpn: UInt) = { 204 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 205 val set = genPtwL2SetIdx(vpn) 206 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 207 l2h(set) 208 } 209 210 211 212 // l3: level 2 leaf pte of 4KB pages 213 val l3 = Module(new SRAMTemplate( 214 l3EntryType, 215 set = l2tlbParams.l3nSets, 216 way = l2tlbParams.l3nWays, 217 singlePort = sramSinglePort 218 )) 219 val l3v = RegInit(0.U((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 220 val l3g = Reg(UInt((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 221 val l3h = Reg(Vec(l2tlbParams.l3nSets, Vec(l2tlbParams.l3nWays, UInt(2.W)))) 222 def getl3vSet(vpn: UInt) = { 223 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 224 val set = genPtwL3SetIdx(vpn) 225 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 226 val l3vVec = l3v.asTypeOf(Vec(l2tlbParams.l3nSets, UInt(l2tlbParams.l3nWays.W))) 227 l3vVec(set) 228 } 229 def getl3hSet(vpn: UInt) = { 230 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 231 val set = genPtwL3SetIdx(vpn) 232 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 233 l3h(set) 234 } 235 236 // sp: level 0/1 leaf pte of 1GB/2MB super pages 237 val sp = Reg(Vec(l2tlbParams.spSize, new PtwEntry(tagLen = SPTagLen, hasPerm = true, hasLevel = true))) 238 val spv = RegInit(0.U(l2tlbParams.spSize.W)) 239 val spg = Reg(UInt(l2tlbParams.spSize.W)) 240 val spasids = sp.map(_.asid) 241 val spvmids = sp.map(_.vmid) 242 val sph = Reg(Vec(l2tlbParams.spSize, UInt(2.W))) 243 244 // Access Perf 245 val l1AccessPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 246 val l2AccessPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 247 val l3AccessPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 248 val spAccessPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 249 l1AccessPerf.map(_ := false.B) 250 l2AccessPerf.map(_ := false.B) 251 l3AccessPerf.map(_ := false.B) 252 spAccessPerf.map(_ := false.B) 253 254 255 256 def vpn_match(vpn1: UInt, vpn2: UInt, level: Int) = { 257 (vpn1(vpnLen-1, vpnnLen*(2-level)+3) === vpn2(vpnLen-1, vpnnLen*(2-level)+3)) 258 } 259 // NOTE: not actually bypassed, just check if hit, re-access the page cache 260 def refill_bypass(vpn: UInt, level: Int, h_search: UInt) = { 261 val change_h = MuxLookup(h_search, noS2xlate, Seq( 262 allStage -> onlyStage1, 263 onlyStage1 -> onlyStage1, 264 onlyStage2 -> onlyStage2 265 )) 266 val refill_vpn = io.refill.bits.req_info_dup(0).vpn 267 io.refill.valid && (level.U === io.refill.bits.level_dup(0)) && vpn_match(refill_vpn, vpn, level) && change_h === io.refill.bits.req_info_dup(0).s2xlate 268 } 269 270 val vpn_search = stageReq.bits.req_info.vpn 271 val h_search = MuxLookup(stageReq.bits.req_info.s2xlate, noS2xlate, Seq( 272 allStage -> onlyStage1, 273 onlyStage1 -> onlyStage1, 274 onlyStage2 -> onlyStage2 275 )) 276 // l1 277 val ptwl1replace = ReplacementPolicy.fromString(l2tlbParams.l1Replacer, l2tlbParams.l1Size) 278 val (l1Hit, l1HitPPN, l1Pre) = { 279 val hitVecT = l1.zipWithIndex.map { 280 case (e, i) => (e.hit(vpn_search, io.csr_dup(0).satp.asid, io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate) 281 && l1v(i) && h_search === l1h(i)) 282 } 283 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 284 285 // stageDelay, but check for l1 286 val hitPPN = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.ppn)), stageDelay_valid_1cycle) 287 val hitPre = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.prefetch)), stageDelay_valid_1cycle) 288 val hit = DataHoldBypass(ParallelOR(hitVec), stageDelay_valid_1cycle) 289 290 when (hit && stageDelay_valid_1cycle) { ptwl1replace.access(OHToUInt(hitVec)) } 291 292 l1AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageDelay_valid_1cycle} 293 for (i <- 0 until l2tlbParams.l1Size) { 294 XSDebug(stageReq.fire, p"[l1] l1(${i.U}) ${l1(i)} hit:${l1(i).hit(vpn_search, Mux(h_search =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate)}\n") 295 } 296 XSDebug(stageReq.fire, p"[l1] l1v:${Binary(l1v)} hitVecT:${Binary(VecInit(hitVecT).asUInt)}\n") 297 XSDebug(stageDelay(0).valid, p"[l1] l1Hit:${hit} l1HitPPN:0x${Hexadecimal(hitPPN)} hitVec:${VecInit(hitVec).asUInt}\n") 298 299 VecInit(hitVecT).suggestName(s"l1_hitVecT") 300 VecInit(hitVec).suggestName(s"l1_hitVec") 301 302 // synchronize with other entries with RegEnable 303 (RegEnable(hit, stageDelay(1).fire), 304 RegEnable(hitPPN, stageDelay(1).fire), 305 RegEnable(hitPre, stageDelay(1).fire)) 306 } 307 308 // l2 309 val ptwl2replace = ReplacementPolicy.fromString(l2tlbParams.l2Replacer,l2tlbParams.l2nWays,l2tlbParams.l2nSets) 310 val (l2Hit, l2HitPPN, l2Pre, l2eccError) = { 311 val ridx = genPtwL2SetIdx(vpn_search) 312 l2.io.r.req.valid := stageReq.fire 313 l2.io.r.req.bits.apply(setIdx = ridx) 314 val vVec_req = getl2vSet(vpn_search) 315 val hVec_req = getl2hSet(vpn_search) 316 317 // delay one cycle after sram read 318 val delay_vpn = stageDelay(0).bits.req_info.vpn 319 val delay_h = MuxLookup(stageDelay(0).bits.req_info.s2xlate, noS2xlate, Seq( 320 allStage -> onlyStage1, 321 onlyStage1 -> onlyStage1, 322 onlyStage2 -> onlyStage2 323 )) 324 val data_resp = DataHoldBypass(l2.io.r.resp.data, stageDelay_valid_1cycle) 325 val vVec_delay = RegEnable(vVec_req, stageReq.fire) 326 val hVec_delay = RegEnable(hVec_req, stageReq.fire) 327 val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).zip(hVec_delay).map { case ((wayData, v), h) => 328 wayData.entries.hit(delay_vpn, io.csr_dup(1).satp.asid, io.csr_dup(1).hgatp.asid, s2xlate = delay_h =/= noS2xlate) && v && (delay_h === h)}) 329 330 // check hit and ecc 331 val check_vpn = stageCheck(0).bits.req_info.vpn 332 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 333 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools 334 335 val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire) 336 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 337 val hitWayData = hitWayEntry.entries 338 val hit = ParallelOR(hitVec) 339 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l2nWays).map(_.U(log2Up(l2tlbParams.l2nWays).W))) 340 val eccError = hitWayEntry.decode() 341 342 ridx.suggestName(s"l2_ridx") 343 ramDatas.suggestName(s"l2_ramDatas") 344 hitVec.suggestName(s"l2_hitVec") 345 hitWayData.suggestName(s"l2_hitWayData") 346 hitWay.suggestName(s"l2_hitWay") 347 348 when (hit && stageCheck_valid_1cycle) { ptwl2replace.access(genPtwL2SetIdx(check_vpn), hitWay) } 349 350 l2AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 351 XSDebug(stageDelay_valid_1cycle, p"[l2] ridx:0x${Hexadecimal(ridx)}\n") 352 for (i <- 0 until l2tlbParams.l2nWays) { 353 XSDebug(stageCheck_valid_1cycle, p"[l2] ramDatas(${i.U}) ${ramDatas(i)} l2v:${vVec(i)} hit:${hit}\n") 354 } 355 XSDebug(stageCheck_valid_1cycle, p"[l2] l2Hit:${hit} l2HitPPN:0x${Hexadecimal(hitWayData.ppns(genPtwL2SectorIdx(check_vpn)))} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${vVec}\n") 356 357 (hit, hitWayData.ppns(genPtwL2SectorIdx(check_vpn)), hitWayData.prefetch, eccError) 358 } 359 360 // l3 361 val ptwl3replace = ReplacementPolicy.fromString(l2tlbParams.l3Replacer,l2tlbParams.l3nWays,l2tlbParams.l3nSets) 362 val (l3Hit, l3HitData, l3Pre, l3eccError) = { 363 val ridx = genPtwL3SetIdx(vpn_search) 364 l3.io.r.req.valid := stageReq.fire 365 l3.io.r.req.bits.apply(setIdx = ridx) 366 val vVec_req = getl3vSet(vpn_search) 367 val hVec_req = getl3hSet(vpn_search) 368 369 // delay one cycle after sram read 370 val delay_vpn = stageDelay(0).bits.req_info.vpn 371 val delay_h = MuxLookup(stageDelay(0).bits.req_info.s2xlate, noS2xlate, Seq( 372 allStage -> onlyStage1, 373 onlyStage1 -> onlyStage1, 374 onlyStage2 -> onlyStage2 375 )) 376 val data_resp = DataHoldBypass(l3.io.r.resp.data, stageDelay_valid_1cycle) 377 val vVec_delay = RegEnable(vVec_req, stageReq.fire) 378 val hVec_delay = RegEnable(hVec_req, stageReq.fire) 379 val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).zip(hVec_delay).map { case ((wayData, v), h) => 380 wayData.entries.hit(delay_vpn, io.csr_dup(2).satp.asid, io.csr_dup(2).hgatp.asid, s2xlate = delay_h =/= noS2xlate) && v && (delay_h === h)}) 381 382 // check hit and ecc 383 val check_vpn = stageCheck(0).bits.req_info.vpn 384 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 385 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools 386 387 val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire) 388 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 389 val hitWayData = hitWayEntry.entries 390 val hitWayEcc = hitWayEntry.ecc 391 val hit = ParallelOR(hitVec) 392 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l3nWays).map(_.U(log2Up(l2tlbParams.l3nWays).W))) 393 val eccError = hitWayEntry.decode() 394 395 when (hit && stageCheck_valid_1cycle) { ptwl3replace.access(genPtwL3SetIdx(check_vpn), hitWay) } 396 397 l3AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 398 XSDebug(stageReq.fire, p"[l3] ridx:0x${Hexadecimal(ridx)}\n") 399 for (i <- 0 until l2tlbParams.l3nWays) { 400 XSDebug(stageCheck_valid_1cycle, p"[l3] ramDatas(${i.U}) ${ramDatas(i)} l3v:${vVec(i)} hit:${hitVec(i)}\n") 401 } 402 XSDebug(stageCheck_valid_1cycle, p"[l3] l3Hit:${hit} l3HitData:${hitWayData} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} v:${vVec}\n") 403 404 ridx.suggestName(s"l3_ridx") 405 ramDatas.suggestName(s"l3_ramDatas") 406 hitVec.suggestName(s"l3_hitVec") 407 hitWay.suggestName(s"l3_hitWay") 408 409 (hit, hitWayData, hitWayData.prefetch, eccError) 410 } 411 val l3HitPPN = l3HitData.ppns 412 val l3HitPerm = l3HitData.perms.getOrElse(0.U.asTypeOf(Vec(PtwL3SectorSize, new PtePermBundle))) 413 val l3HitValid = l3HitData.vs 414 415 // super page 416 val spreplace = ReplacementPolicy.fromString(l2tlbParams.spReplacer, l2tlbParams.spSize) 417 val (spHit, spHitData, spPre, spValid) = { 418 val hitVecT = sp.zipWithIndex.map { case (e, i) => e.hit(vpn_search, Mux(h_search =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate) && spv(i) && (sph(i) === h_search) } 419 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 420 val hitData = ParallelPriorityMux(hitVec zip sp) 421 val hit = ParallelOR(hitVec) 422 423 when (hit && stageDelay_valid_1cycle) { spreplace.access(OHToUInt(hitVec)) } 424 425 spAccessPerf.zip(hitVec).map{ case (s, h) => s := h && stageDelay_valid_1cycle } 426 for (i <- 0 until l2tlbParams.spSize) { 427 XSDebug(stageReq.fire, p"[sp] sp(${i.U}) ${sp(i)} hit:${sp(i).hit(vpn_search, Mux(h_search =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), io.csr_dup(0).hgatp.asid, s2xlate = h_search =/= noS2xlate)} spv:${spv(i)}\n") 428 } 429 XSDebug(stageDelay_valid_1cycle, p"[sp] spHit:${hit} spHitData:${hitData} hitVec:${Binary(VecInit(hitVec).asUInt)}\n") 430 431 VecInit(hitVecT).suggestName(s"sp_hitVecT") 432 VecInit(hitVec).suggestName(s"sp_hitVec") 433 434 (RegEnable(hit, stageDelay(1).fire), 435 RegEnable(hitData, stageDelay(1).fire), 436 RegEnable(hitData.prefetch, stageDelay(1).fire), 437 RegEnable(hitData.v, stageDelay(1).fire)) 438 } 439 val spHitPerm = spHitData.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)) 440 val spHitLevel = spHitData.level.getOrElse(0.U) 441 442 val check_res = Wire(new PageCacheRespBundle) 443 check_res.l1.apply(l1Hit, l1Pre, l1HitPPN) 444 check_res.l2.apply(l2Hit, l2Pre, l2HitPPN, ecc = l2eccError) 445 check_res.l3.apply(l3Hit, l3Pre, l3HitPPN, l3HitPerm, l3eccError, valid = l3HitValid) 446 check_res.sp.apply(spHit, spPre, spHitData.ppn, spHitPerm, false.B, spHitLevel, spValid) 447 448 val resp_res = Reg(new PageCacheRespBundle) 449 when (stageCheck(1).fire) { resp_res := check_res } 450 451 // stageResp bypass 452 val bypassed = Wire(Vec(3, Bool())) 453 bypassed.indices.foreach(i => 454 bypassed(i) := stageResp.bits.bypassed(i) || 455 ValidHoldBypass(refill_bypass(vpn_search, i, stageResp.bits.req_info.s2xlate), 456 OneCycleValid(stageCheck(1).fire, false.B) || io.refill.valid) 457 ) 458 459 val isAllStage = stageResp.bits.req_info.s2xlate === allStage 460 val isOnlyStage2 = stageResp.bits.req_info.s2xlate === onlyStage2 461 val stage1Hit = (resp_res.l3.hit || resp_res.sp.hit) && isAllStage 462 io.resp.bits.req_info := stageResp.bits.req_info 463 io.resp.bits.isFirst := stageResp.bits.isFirst 464 io.resp.bits.hit := (resp_res.l3.hit || resp_res.sp.hit) && !isAllStage 465 io.resp.bits.bypassed := (bypassed(2) || (bypassed(1) && !resp_res.l2.hit) || (bypassed(0) && !resp_res.l1.hit)) && !isAllStage && !isOnlyStage2 466 io.resp.bits.prefetch := resp_res.l3.pre && resp_res.l3.hit || resp_res.sp.pre && resp_res.sp.hit 467 io.resp.bits.toFsm.l1Hit := resp_res.l1.hit && !stage1Hit && !isOnlyStage2 468 io.resp.bits.toFsm.l2Hit := resp_res.l2.hit && !stage1Hit && !isOnlyStage2 469 io.resp.bits.toFsm.ppn := Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l1.ppn) 470 io.resp.bits.toFsm.stage1Hit := stage1Hit 471 472 io.resp.bits.isHptw := stageResp.bits.isHptw 473 io.resp.bits.toHptw.id := stageResp.bits.hptwId 474 io.resp.bits.toHptw.l1Hit := resp_res.l1.hit 475 io.resp.bits.toHptw.l2Hit := resp_res.l2.hit 476 io.resp.bits.toHptw.ppn := Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l1.ppn) 477 val idx = stageResp.bits.req_info.vpn(2, 0) 478 io.resp.bits.toHptw.resp.entry.tag := stageResp.bits.req_info.vpn 479 io.resp.bits.toHptw.resp.entry.asid := DontCare 480 io.resp.bits.toHptw.resp.entry.vmid.map(_ := io.csr_dup(0).hgatp.asid) 481 io.resp.bits.toHptw.resp.entry.level.map(_ := Mux(resp_res.l3.hit, 2.U, resp_res.sp.level)) 482 io.resp.bits.toHptw.resp.entry.prefetch := from_pre(stageResp.bits.req_info.source) 483 io.resp.bits.toHptw.resp.entry.ppn := Mux(resp_res.l3.hit, resp_res.l3.ppn(idx), resp_res.sp.ppn) 484 io.resp.bits.toHptw.resp.entry.perm.map(_ := Mux(resp_res.l3.hit, resp_res.l3.perm(idx), resp_res.sp.perm)) 485 io.resp.bits.toHptw.resp.entry.v := Mux(resp_res.l3.hit, resp_res.l3.v(idx), resp_res.sp.v) 486 io.resp.bits.toHptw.resp.gpf := !io.resp.bits.toHptw.resp.entry.v 487 io.resp.bits.toHptw.resp.gaf := false.B 488 489 io.resp.bits.toTlb.entry.map(_.tag := stageResp.bits.req_info.vpn(vpnLen - 1, 3)) 490 io.resp.bits.toTlb.entry.map(_.asid := Mux(stageResp.bits.req_info.hasS2xlate(), io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid)) // DontCare 491 io.resp.bits.toTlb.entry.map(_.vmid.map(_ := io.csr_dup(0).hgatp.asid)) 492 io.resp.bits.toTlb.entry.map(_.level.map(_ := Mux(resp_res.l3.hit, 2.U, resp_res.sp.level))) 493 io.resp.bits.toTlb.entry.map(_.prefetch := from_pre(stageResp.bits.req_info.source)) 494 for (i <- 0 until tlbcontiguous) { 495 io.resp.bits.toTlb.entry(i).ppn := Mux(resp_res.l3.hit, resp_res.l3.ppn(i)(ppnLen - 1, sectortlbwidth), resp_res.sp.ppn(ppnLen - 1, sectortlbwidth)) 496 io.resp.bits.toTlb.entry(i).ppn_low := Mux(resp_res.l3.hit, resp_res.l3.ppn(i)(sectortlbwidth - 1, 0), resp_res.sp.ppn(sectortlbwidth - 1, 0)) 497 io.resp.bits.toTlb.entry(i).perm.map(_ := Mux(resp_res.l3.hit, resp_res.l3.perm(i), resp_res.sp.perm)) 498 io.resp.bits.toTlb.entry(i).v := Mux(resp_res.l3.hit, resp_res.l3.v(i), resp_res.sp.v) 499 io.resp.bits.toTlb.entry(i).pf := !io.resp.bits.toTlb.entry(i).v 500 io.resp.bits.toTlb.entry(i).af := false.B 501 } 502 io.resp.bits.toTlb.pteidx := UIntToOH(stageResp.bits.req_info.vpn(2, 0)).asBools 503 io.resp.bits.toTlb.not_super := Mux(resp_res.l3.hit, true.B, false.B) 504 io.resp.valid := stageResp.valid 505 XSError(stageResp.valid && resp_res.l3.hit && resp_res.sp.hit, "normal page and super page both hit") 506 XSError(stageResp.valid && io.resp.bits.hit && bypassed(2), "page cache, bypassed but hit") 507 508 // refill Perf 509 val l1RefillPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 510 val l2RefillPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 511 val l3RefillPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 512 val spRefillPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 513 l1RefillPerf.map(_ := false.B) 514 l2RefillPerf.map(_ := false.B) 515 l3RefillPerf.map(_ := false.B) 516 spRefillPerf.map(_ := false.B) 517 518 // refill 519 l2.io.w.req <> DontCare 520 l3.io.w.req <> DontCare 521 l2.io.w.req.valid := false.B 522 l3.io.w.req.valid := false.B 523 524 val memRdata = refill.ptes 525 val memPtes = (0 until (l2tlbParams.blockBytes/(XLEN/8))).map(i => memRdata((i+1)*XLEN-1, i*XLEN).asTypeOf(new PteBundle)) 526 val memSelData = io.refill.bits.sel_pte_dup 527 val memPte = memSelData.map(a => a.asTypeOf(new PteBundle)) 528 529 // TODO: handle sfenceLatch outsize 530 when (!flush_dup(0) && refill.levelOH.l1 && !memPte(0).isLeaf() && !memPte(0).isPf(refill.level_dup(0)) && !memPte(0).isAf()) { 531 // val refillIdx = LFSR64()(log2Up(l2tlbParams.l1Size)-1,0) // TODO: may be LRU 532 val refillIdx = replaceWrapper(l1v, ptwl1replace.way) 533 refillIdx.suggestName(s"PtwL1RefillIdx") 534 val rfOH = UIntToOH(refillIdx) 535 l1(refillIdx).refill( 536 refill.req_info_dup(0).vpn, 537 Mux(refill.req_info_dup(0).s2xlate =/= noS2xlate, io.csr_dup(0).vsatp.asid, io.csr_dup(0).satp.asid), 538 io.csr_dup(0).hgatp.asid, 539 memSelData(0), 540 0.U, 541 refill_prefetch_dup(0) 542 ) 543 ptwl1replace.access(refillIdx) 544 l1v := l1v | rfOH 545 l1g := (l1g & ~rfOH) | Mux(memPte(0).perm.g, rfOH, 0.U) 546 l1h(refillIdx) := refill.req_info_dup(0).s2xlate 547 548 for (i <- 0 until l2tlbParams.l1Size) { 549 l1RefillPerf(i) := i.U === refillIdx 550 } 551 552 XSDebug(p"[l1 refill] refillIdx:${refillIdx} refillEntry:${l1(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, io.csr_dup(0).satp.asid, memSelData(0), 0.U, prefetch = refill_prefetch_dup(0))}\n") 553 XSDebug(p"[l1 refill] l1v:${Binary(l1v)}->${Binary(l1v | rfOH)} l1g:${Binary(l1g)}->${Binary((l1g & ~rfOH) | Mux(memPte(0).perm.g, rfOH, 0.U))}\n") 554 555 refillIdx.suggestName(s"l1_refillIdx") 556 rfOH.suggestName(s"l1_rfOH") 557 } 558 559 when (!flush_dup(1) && refill.levelOH.l2 && !memPte(1).isLeaf() && !memPte(1).isPf(refill.level_dup(1)) && !memPte(1).isAf()) { 560 val refillIdx = genPtwL2SetIdx(refill.req_info_dup(1).vpn) 561 val victimWay = replaceWrapper(getl2vSet(refill.req_info_dup(1).vpn), ptwl2replace.way(refillIdx)) 562 val victimWayOH = UIntToOH(victimWay) 563 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 564 val wdata = Wire(l2EntryType) 565 wdata.gen( 566 vpn = refill.req_info_dup(1).vpn, 567 asid = Mux(refill.req_info_dup(1).s2xlate =/= noS2xlate, io.csr_dup(1).vsatp.asid, io.csr_dup(1).satp.asid), 568 vmid = io.csr_dup(1).hgatp.asid, 569 data = memRdata, 570 levelUInt = 1.U, 571 refill_prefetch_dup(1) 572 ) 573 l2.io.w.apply( 574 valid = true.B, 575 setIdx = refillIdx, 576 data = wdata, 577 waymask = victimWayOH 578 ) 579 ptwl2replace.access(refillIdx, victimWay) 580 l2v := l2v | rfvOH 581 l2g := l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 582 l2h(refillIdx)(victimWay) := refill.req_info_dup(1).s2xlate 583 584 for (i <- 0 until l2tlbParams.l2nWays) { 585 l2RefillPerf(i) := i.U === victimWay 586 } 587 588 XSDebug(p"[l2 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 589 XSDebug(p"[l2 refill] refilldata:0x${wdata}\n") 590 XSDebug(p"[l2 refill] l2v:${Binary(l2v)} -> ${Binary(l2v | rfvOH)}\n") 591 XSDebug(p"[l2 refill] l2g:${Binary(l2g)} -> ${Binary(l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 592 593 refillIdx.suggestName(s"l2_refillIdx") 594 victimWay.suggestName(s"l2_victimWay") 595 victimWayOH.suggestName(s"l2_victimWayOH") 596 rfvOH.suggestName(s"l2_rfvOH") 597 } 598 599 when (!flush_dup(2) && refill.levelOH.l3 && !memPte(2).isAf()) { 600 val refillIdx = genPtwL3SetIdx(refill.req_info_dup(2).vpn) 601 val victimWay = replaceWrapper(getl3vSet(refill.req_info_dup(2).vpn), ptwl3replace.way(refillIdx)) 602 val victimWayOH = UIntToOH(victimWay) 603 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 604 val wdata = Wire(l3EntryType) 605 wdata.gen( 606 vpn = refill.req_info_dup(2).vpn, 607 asid = Mux(refill.req_info_dup(2).s2xlate =/= noS2xlate, io.csr_dup(2).vsatp.asid, io.csr_dup(2).satp.asid), 608 vmid = io.csr_dup(2).hgatp.asid, 609 data = memRdata, 610 levelUInt = 2.U, 611 refill_prefetch_dup(2) 612 ) 613 l3.io.w.apply( 614 valid = true.B, 615 setIdx = refillIdx, 616 data = wdata, 617 waymask = victimWayOH 618 ) 619 ptwl3replace.access(refillIdx, victimWay) 620 l3v := l3v | rfvOH 621 l3g := l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 622 l3h(refillIdx)(victimWay) := refill.req_info_dup(2).s2xlate 623 624 for (i <- 0 until l2tlbParams.l3nWays) { 625 l3RefillPerf(i) := i.U === victimWay 626 } 627 628 XSDebug(p"[l3 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 629 XSDebug(p"[l3 refill] refilldata:0x${wdata}\n") 630 XSDebug(p"[l3 refill] l3v:${Binary(l3v)} -> ${Binary(l3v | rfvOH)}\n") 631 XSDebug(p"[l3 refill] l3g:${Binary(l3g)} -> ${Binary(l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 632 633 refillIdx.suggestName(s"l3_refillIdx") 634 victimWay.suggestName(s"l3_victimWay") 635 victimWayOH.suggestName(s"l3_victimWayOH") 636 rfvOH.suggestName(s"l3_rfvOH") 637 } 638 639 640 // misc entries: super & invalid 641 when (!flush_dup(0) && refill.levelOH.sp && (memPte(0).isLeaf() || memPte(0).isPf(refill.level_dup(0))) && !memPte(0).isAf()) { 642 val refillIdx = spreplace.way// LFSR64()(log2Up(l2tlbParams.spSize)-1,0) // TODO: may be LRU 643 val rfOH = UIntToOH(refillIdx) 644 sp(refillIdx).refill( 645 refill.req_info_dup(0).vpn, 646 io.csr_dup(0).satp.asid, 647 io.csr_dup(0).hgatp.asid, 648 memSelData(0), 649 refill.level_dup(2), 650 refill_prefetch_dup(0), 651 !memPte(0).isPf(refill.level_dup(0)), 652 ) 653 spreplace.access(refillIdx) 654 spv := spv | rfOH 655 spg := spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U) 656 sph(refillIdx) := refill.req_info_dup(0).s2xlate 657 658 for (i <- 0 until l2tlbParams.spSize) { 659 spRefillPerf(i) := i.U === refillIdx 660 } 661 662 XSDebug(p"[sp refill] refillIdx:${refillIdx} refillEntry:${sp(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, io.csr_dup(0).satp.asid, memSelData(0), refill.level_dup(0), refill_prefetch_dup(0))}\n") 663 XSDebug(p"[sp refill] spv:${Binary(spv)}->${Binary(spv | rfOH)} spg:${Binary(spg)}->${Binary(spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U))}\n") 664 665 refillIdx.suggestName(s"sp_refillIdx") 666 rfOH.suggestName(s"sp_rfOH") 667 } 668 669 val l2eccFlush = resp_res.l2.ecc && stageResp_valid_1cycle_dup(0) // RegNext(l2eccError, init = false.B) 670 val l3eccFlush = resp_res.l3.ecc && stageResp_valid_1cycle_dup(1) // RegNext(l3eccError, init = false.B) 671 val eccVpn = stageResp.bits.req_info.vpn 672 673 XSError(l2eccFlush, "l2tlb.cache.l2 ecc error. Should not happen at sim stage") 674 XSError(l3eccFlush, "l2tlb.cache.l3 ecc error. Should not happen at sim stage") 675 when (l2eccFlush) { 676 val flushSetIdxOH = UIntToOH(genPtwL2SetIdx(eccVpn)) 677 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l2nWays, a.asUInt) }).asUInt 678 l2v := l2v & ~flushMask 679 l2g := l2g & ~flushMask 680 } 681 682 when (l3eccFlush) { 683 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(eccVpn)) 684 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 685 l3v := l3v & ~flushMask 686 l3g := l3g & ~flushMask 687 } 688 689 // sfence for l3 690 val sfence_valid_l3 = sfence_dup(3).valid && !sfence_dup(3).bits.hg && !sfence_dup(3).bits.hv 691 when (sfence_valid_l3) { 692 val l3hhit = VecInit(l3h.flatMap(_.map(_ === onlyStage1 && io.csr_dup(0).priv.virt || !io.csr_dup(0).priv.virt))).asUInt 693 val sfence_vpn = sfence_dup(3).bits.addr(sfence_dup(3).bits.addr.getWidth-1, offLen) 694 when (sfence_dup(3).bits.rs1/*va*/) { 695 when (sfence_dup(3).bits.rs2) { 696 // all va && all asid 697 l3v := l3v & ~l3hhit 698 } .otherwise { 699 // all va && specific asid except global 700 l3v := l3v & l3g & ~l3hhit 701 } 702 } .otherwise { 703 // val flushMask = UIntToOH(genTlbL2Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen))) 704 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(sfence_vpn)) 705 // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l3nWays, _.asUInt))).asUInt 706 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 707 flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH") 708 flushMask.suggestName(s"sfence_nrs1_flushMask") 709 710 when (sfence_dup(3).bits.rs2) { 711 // specific leaf of addr && all asid 712 l3v := l3v & ~flushMask & ~l3hhit 713 } .otherwise { 714 // specific leaf of addr && specific asid 715 l3v := l3v & (~flushMask | l3g | ~l3hhit) 716 } 717 } 718 } 719 720 // sfence for virtualization and hfencev, simple implementation for l3 721 val hfencev_valid_l3 = sfence_dup(3).valid && sfence_dup(3).bits.hv 722 when(hfencev_valid_l3) { 723 val flushMask = VecInit(l3h.flatMap(_.map(_ === onlyStage1))).asUInt 724 l3v := l3v & ~flushMask // all VS-stage l3 pte 725 } 726 727 // hfenceg, simple implementation for l3 728 val hfenceg_valid_l3 = sfence_dup(3).valid && sfence_dup(3).bits.hg 729 when(hfenceg_valid_l3) { 730 val flushMask = VecInit(l3h.flatMap(_.map(_ === onlyStage2))).asUInt 731 l3v := l3v & ~flushMask // all G-stage l3 pte 732 } 733 734 735 val l1asidhit = VecInit(l1asids.map(_ === sfence_dup(0).bits.id)).asUInt 736 val spasidhit = VecInit(spasids.map(_ === sfence_dup(0).bits.id)).asUInt 737 val sfence_valid = sfence_dup(0).valid && !sfence_dup(0).bits.hg && !sfence_dup(0).bits.hv 738 when (sfence_valid) { 739 val l1vmidhit = VecInit(l1vmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt 740 val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt 741 val l1hhit = VecInit(l1h.map(_ === onlyStage1 && io.csr_dup(0).priv.virt || !io.csr_dup(0).priv.virt)).asUInt 742 val sphhit = VecInit(sph.map(_ === onlyStage1 && io.csr_dup(0).priv.virt || !io.csr_dup(0).priv.virt)).asUInt 743 val l2hhit = VecInit(l2h.flatMap(_.map(_ === onlyStage1 && io.csr_dup(0).priv.virt || !io.csr_dup(0).priv.virt))).asUInt 744 val sfence_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen) 745 val l2h_set = getl2hSet(sfence_vpn) 746 747 when (sfence_dup(0).bits.rs1/*va*/) { 748 when (sfence_dup(0).bits.rs2) { 749 // all va && all asid 750 l2v := l2v & ~l2hhit 751 l1v := l1v & ~(l1hhit & VecInit(UIntToOH(l1vmidhit).asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt) 752 spv := spv & ~(l2hhit & VecInit(UIntToOH(spvmidhit).asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt) 753 } .otherwise { 754 // all va && specific asid except global 755 l2v := l2v & (l2g | ~l2hhit) 756 l1v := l1v & ~(~l1g & l1hhit & l1asidhit & VecInit(UIntToOH(l1vmidhit).asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt) 757 spv := spv & ~(~spg & sphhit & spasidhit & VecInit(UIntToOH(spvmidhit).asBools.map{a => io.csr_dup(0).priv.virt && a || !io.csr_dup(0).priv.virt}).asUInt) 758 } 759 } .otherwise { 760 when (sfence_dup(0).bits.rs2) { 761 // specific leaf of addr && all asid 762 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, ignoreAsid = true, s2xlate = io.csr_dup(0).priv.virt))).asUInt) 763 } .otherwise { 764 // specific leaf of addr && specific asid 765 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, s2xlate = io.csr_dup(0).priv.virt))).asUInt | spg) 766 } 767 } 768 } 769 770 val hfencev_valid = sfence_dup(0).valid && sfence_dup(0).bits.hv 771 when (hfencev_valid) { 772 val l1vmidhit = VecInit(l1vmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt 773 val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === io.csr_dup(0).hgatp.asid)).asUInt 774 val l1hhit = VecInit(l1h.map(_ === onlyStage1)).asUInt 775 val sphhit = VecInit(sph.map(_ === onlyStage1)).asUInt 776 val l2hhit = VecInit(l2h.flatMap(_.map(_ === onlyStage1))).asUInt 777 val hfencev_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen) 778 when(sfence_dup(0).bits.rs1) { 779 when(sfence_dup(0).bits.rs2) { 780 l2v := l2v & ~l2hhit 781 l1v := l1v & ~(l1hhit & l1vmidhit) 782 spv := spv & ~(l2hhit & spvmidhit) 783 }.otherwise { 784 l2v := l2v & (l2g | ~l2hhit) 785 l1v := l1v & ~(~l1g & l1hhit & l1asidhit & l1vmidhit) 786 spv := spv & ~(~spg & sphhit & spasidhit & spvmidhit) 787 } 788 }.otherwise { 789 when(sfence_dup(0).bits.rs2) { 790 spv := spv & (~VecInit(sp.map(_.hit(hfencev_vpn, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, ignoreAsid = true, s2xlate = true.B))).asUInt) 791 }.otherwise { 792 spv := spv & (~VecInit(sp.map(_.hit(hfencev_vpn, sfence_dup(0).bits.id, io.csr_dup(0).hgatp.asid, s2xlate = true.B))).asUInt | spg) 793 } 794 } 795 } 796 797 798 val hfenceg_valid = sfence_dup(0).valid && sfence_dup(0).bits.hg 799 when(hfenceg_valid) { 800 val l1vmidhit = VecInit(l1vmids.map(_.getOrElse(0.U) === sfence_dup(0).bits.id)).asUInt 801 val spvmidhit = VecInit(spvmids.map(_.getOrElse(0.U) === sfence_dup(0).bits.id)).asUInt 802 val l1hhit = VecInit(l1h.map(_ === onlyStage2)).asUInt 803 val sphhit = VecInit(sph.map(_ === onlyStage2)).asUInt 804 val l2hhit = VecInit(l2h.flatMap(_.map(_ === onlyStage2))).asUInt 805 val hfenceg_gvpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth - 1, offLen) 806 when(sfence_dup(0).bits.rs1) { 807 when(sfence_dup(0).bits.rs2) { 808 l2v := l2v & ~l2hhit 809 l1v := l1v & ~l1hhit 810 spv := spv & ~sphhit 811 }.otherwise { 812 l2v := l2v & ~l2hhit 813 l1v := l1v & ~(l1hhit & l1vmidhit) 814 spv := spv & ~(sphhit & spvmidhit) 815 } 816 }.otherwise { 817 when(sfence_dup(0).bits.rs2) { 818 spv := spv & (~VecInit(sp.map(_.hit(hfenceg_gvpn, 0.U, sfence_dup(0).bits.id, ignoreAsid = true, s2xlate = true.B))).asUInt) 819 }.otherwise { 820 spv := spv & (~VecInit(sp.map(_.hit(hfenceg_gvpn, 0.U, sfence_dup(0).bits.id, ignoreAsid = true, s2xlate = false.B))).asUInt) 821 } 822 } 823 } 824 825 def InsideStageConnect(in: DecoupledIO[PtwCacheReq], out: DecoupledIO[PtwCacheReq], inFire: Bool): Unit = { 826 in.ready := !in.valid || out.ready 827 out.valid := in.valid 828 out.bits := in.bits 829 out.bits.bypassed.zip(in.bits.bypassed).zipWithIndex.map{ case (b, i) => 830 val bypassed_reg = Reg(Bool()) 831 val bypassed_wire = refill_bypass(in.bits.req_info.vpn, i, in.bits.req_info.s2xlate) && io.refill.valid 832 when (inFire) { bypassed_reg := bypassed_wire } 833 .elsewhen (io.refill.valid) { bypassed_reg := bypassed_reg || bypassed_wire } 834 835 b._1 := b._2 || (bypassed_wire || (bypassed_reg && !inFire)) 836 } 837 } 838 839 // Perf Count 840 val resp_l3 = resp_res.l3.hit 841 val resp_sp = resp_res.sp.hit 842 val resp_l1_pre = resp_res.l1.pre 843 val resp_l2_pre = resp_res.l2.pre 844 val resp_l3_pre = resp_res.l3.pre 845 val resp_sp_pre = resp_res.sp.pre 846 val base_valid_access_0 = !from_pre(io.resp.bits.req_info.source) && io.resp.fire 847 XSPerfAccumulate("access", base_valid_access_0) 848 XSPerfAccumulate("l1_hit", base_valid_access_0 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 849 XSPerfAccumulate("l2_hit", base_valid_access_0 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 850 XSPerfAccumulate("l3_hit", base_valid_access_0 && resp_l3) 851 XSPerfAccumulate("sp_hit", base_valid_access_0 && resp_sp) 852 XSPerfAccumulate("pte_hit",base_valid_access_0 && io.resp.bits.hit) 853 854 XSPerfAccumulate("l1_hit_pre", base_valid_access_0 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 855 XSPerfAccumulate("l2_hit_pre", base_valid_access_0 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 856 XSPerfAccumulate("l3_hit_pre", base_valid_access_0 && resp_l3_pre && resp_l3) 857 XSPerfAccumulate("sp_hit_pre", base_valid_access_0 && resp_sp_pre && resp_sp) 858 XSPerfAccumulate("pte_hit_pre",base_valid_access_0 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 859 860 val base_valid_access_1 = from_pre(io.resp.bits.req_info.source) && io.resp.fire 861 XSPerfAccumulate("pre_access", base_valid_access_1) 862 XSPerfAccumulate("pre_l1_hit", base_valid_access_1 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 863 XSPerfAccumulate("pre_l2_hit", base_valid_access_1 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 864 XSPerfAccumulate("pre_l3_hit", base_valid_access_1 && resp_l3) 865 XSPerfAccumulate("pre_sp_hit", base_valid_access_1 && resp_sp) 866 XSPerfAccumulate("pre_pte_hit",base_valid_access_1 && io.resp.bits.hit) 867 868 XSPerfAccumulate("pre_l1_hit_pre", base_valid_access_1 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 869 XSPerfAccumulate("pre_l2_hit_pre", base_valid_access_1 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 870 XSPerfAccumulate("pre_l3_hit_pre", base_valid_access_1 && resp_l3_pre && resp_l3) 871 XSPerfAccumulate("pre_sp_hit_pre", base_valid_access_1 && resp_sp_pre && resp_sp) 872 XSPerfAccumulate("pre_pte_hit_pre",base_valid_access_1 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 873 874 val base_valid_access_2 = stageResp.bits.isFirst && !from_pre(io.resp.bits.req_info.source) && io.resp.fire 875 XSPerfAccumulate("access_first", base_valid_access_2) 876 XSPerfAccumulate("l1_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 877 XSPerfAccumulate("l2_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 878 XSPerfAccumulate("l3_hit_first", base_valid_access_2 && resp_l3) 879 XSPerfAccumulate("sp_hit_first", base_valid_access_2 && resp_sp) 880 XSPerfAccumulate("pte_hit_first",base_valid_access_2 && io.resp.bits.hit) 881 882 XSPerfAccumulate("l1_hit_pre_first", base_valid_access_2 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 883 XSPerfAccumulate("l2_hit_pre_first", base_valid_access_2 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 884 XSPerfAccumulate("l3_hit_pre_first", base_valid_access_2 && resp_l3_pre && resp_l3) 885 XSPerfAccumulate("sp_hit_pre_first", base_valid_access_2 && resp_sp_pre && resp_sp) 886 XSPerfAccumulate("pte_hit_pre_first",base_valid_access_2 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 887 888 val base_valid_access_3 = stageResp.bits.isFirst && from_pre(io.resp.bits.req_info.source) && io.resp.fire 889 XSPerfAccumulate("pre_access_first", base_valid_access_3) 890 XSPerfAccumulate("pre_l1_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 891 XSPerfAccumulate("pre_l2_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 892 XSPerfAccumulate("pre_l3_hit_first", base_valid_access_3 && resp_l3) 893 XSPerfAccumulate("pre_sp_hit_first", base_valid_access_3 && resp_sp) 894 XSPerfAccumulate("pre_pte_hit_first", base_valid_access_3 && io.resp.bits.hit) 895 896 XSPerfAccumulate("pre_l1_hit_pre_first", base_valid_access_3 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 897 XSPerfAccumulate("pre_l2_hit_pre_first", base_valid_access_3 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 898 XSPerfAccumulate("pre_l3_hit_pre_first", base_valid_access_3 && resp_l3_pre && resp_l3) 899 XSPerfAccumulate("pre_sp_hit_pre_first", base_valid_access_3 && resp_sp_pre && resp_sp) 900 XSPerfAccumulate("pre_pte_hit_pre_first",base_valid_access_3 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 901 902 XSPerfAccumulate("rwHarzad", io.req.valid && !io.req.ready) 903 XSPerfAccumulate("out_blocked", io.resp.valid && !io.resp.ready) 904 l1AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1AccessIndex${i}", l) } 905 l2AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2AccessIndex${i}", l) } 906 l3AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3AccessIndex${i}", l) } 907 spAccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPAccessIndex${i}", l) } 908 l1RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1RefillIndex${i}", l) } 909 l2RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2RefillIndex${i}", l) } 910 l3RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3RefillIndex${i}", l) } 911 spRefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPRefillIndex${i}", l) } 912 913 XSPerfAccumulate("l1Refill", Cat(l1RefillPerf).orR) 914 XSPerfAccumulate("l2Refill", Cat(l2RefillPerf).orR) 915 XSPerfAccumulate("l3Refill", Cat(l3RefillPerf).orR) 916 XSPerfAccumulate("spRefill", Cat(spRefillPerf).orR) 917 XSPerfAccumulate("l1Refill_pre", Cat(l1RefillPerf).orR && refill_prefetch_dup(0)) 918 XSPerfAccumulate("l2Refill_pre", Cat(l2RefillPerf).orR && refill_prefetch_dup(0)) 919 XSPerfAccumulate("l3Refill_pre", Cat(l3RefillPerf).orR && refill_prefetch_dup(0)) 920 XSPerfAccumulate("spRefill_pre", Cat(spRefillPerf).orR && refill_prefetch_dup(0)) 921 922 // debug 923 XSDebug(sfence_dup(0).valid, p"[sfence] original v and g vector:\n") 924 XSDebug(sfence_dup(0).valid, p"[sfence] l1v:${Binary(l1v)}\n") 925 XSDebug(sfence_dup(0).valid, p"[sfence] l2v:${Binary(l2v)}\n") 926 XSDebug(sfence_dup(0).valid, p"[sfence] l3v:${Binary(l3v)}\n") 927 XSDebug(sfence_dup(0).valid, p"[sfence] l3g:${Binary(l3g)}\n") 928 XSDebug(sfence_dup(0).valid, p"[sfence] spv:${Binary(spv)}\n") 929 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] new v and g vector:\n") 930 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l1v:${Binary(l1v)}\n") 931 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l2v:${Binary(l2v)}\n") 932 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3v:${Binary(l3v)}\n") 933 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3g:${Binary(l3g)}\n") 934 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] spv:${Binary(spv)}\n") 935 936 val perfEvents = Seq( 937 ("access ", base_valid_access_0 ), 938 ("l1_hit ", l1Hit ), 939 ("l2_hit ", l2Hit ), 940 ("l3_hit ", l3Hit ), 941 ("sp_hit ", spHit ), 942 ("pte_hit ", l3Hit || spHit ), 943 ("rwHarzad ", io.req.valid && !io.req.ready ), 944 ("out_blocked ", io.resp.valid && !io.resp.ready), 945 ) 946 generatePerfEvent() 947} 948