1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import chisel3.internal.naming.chiselName 23import xiangshan._ 24import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 25import utils._ 26import utility._ 27import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 28import freechips.rocketchip.tilelink._ 29 30/* ptw cache caches the page table of all the three layers 31 * ptw cache resp at next cycle 32 * the cache should not be blocked 33 * when miss queue if full, just block req outside 34 */ 35 36class PageCachePerPespBundle(implicit p: Parameters) extends PtwBundle { 37 val hit = Bool() 38 val pre = Bool() 39 val ppn = UInt(ppnLen.W) 40 val perm = new PtePermBundle() 41 val ecc = Bool() 42 val level = UInt(2.W) 43 val v = Bool() 44 45 def apply(hit: Bool, pre: Bool, ppn: UInt, perm: PtePermBundle = 0.U.asTypeOf(new PtePermBundle()), 46 ecc: Bool = false.B, level: UInt = 0.U, valid: Bool = true.B) { 47 this.hit := hit && !ecc 48 this.pre := pre 49 this.ppn := ppn 50 this.perm := perm 51 this.ecc := ecc && hit 52 this.level := level 53 this.v := valid 54 } 55} 56 57class PageCacheMergePespBundle(implicit p: Parameters) extends PtwBundle { 58 assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 59 val hit = Bool() 60 val pre = Bool() 61 val ppn = Vec(tlbcontiguous, UInt(ppnLen.W)) 62 val perm = Vec(tlbcontiguous, new PtePermBundle()) 63 val ecc = Bool() 64 val level = UInt(2.W) 65 val v = Vec(tlbcontiguous, Bool()) 66 67 def apply(hit: Bool, pre: Bool, ppn: Vec[UInt], perm: Vec[PtePermBundle] = Vec(tlbcontiguous, 0.U.asTypeOf(new PtePermBundle())), 68 ecc: Bool = false.B, level: UInt = 0.U, valid: Vec[Bool] = Vec(tlbcontiguous, true.B)) { 69 this.hit := hit && !ecc 70 this.pre := pre 71 this.ppn := ppn 72 this.perm := perm 73 this.ecc := ecc && hit 74 this.level := level 75 this.v := valid 76 } 77} 78 79class PageCacheRespBundle(implicit p: Parameters) extends PtwBundle { 80 val l1 = new PageCachePerPespBundle 81 val l2 = new PageCachePerPespBundle 82 val l3 = new PageCacheMergePespBundle 83 val sp = new PageCachePerPespBundle 84} 85 86class PtwCacheReq(implicit p: Parameters) extends PtwBundle { 87 val req_info = new L2TlbInnerBundle() 88 val isFirst = Bool() 89 val bypassed = Vec(3, Bool()) 90} 91 92class PtwCacheIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 93 val req = Flipped(DecoupledIO(new PtwCacheReq())) 94 val resp = DecoupledIO(new Bundle { 95 val req_info = new L2TlbInnerBundle() 96 val isFirst = Bool() 97 val hit = Bool() 98 val prefetch = Bool() // is the entry fetched by prefetch 99 val bypassed = Bool() 100 val toFsm = new Bundle { 101 val l1Hit = Bool() 102 val l2Hit = Bool() 103 val ppn = UInt(ppnLen.W) 104 } 105 val toTlb = new PtwMergeResp() 106 }) 107 val refill = Flipped(ValidIO(new Bundle { 108 val ptes = UInt(blockBits.W) 109 val levelOH = new Bundle { 110 // NOTE: levelOH has (Level+1) bits, each stands for page cache entries 111 val sp = Bool() 112 val l3 = Bool() 113 val l2 = Bool() 114 val l1 = Bool() 115 def apply(levelUInt: UInt, valid: Bool) = { 116 sp := RegNext((levelUInt === 0.U || levelUInt === 1.U) && valid, false.B) 117 l3 := RegNext((levelUInt === 2.U) & valid, false.B) 118 l2 := RegNext((levelUInt === 1.U) & valid, false.B) 119 l1 := RegNext((levelUInt === 0.U) & valid, false.B) 120 } 121 } 122 // duplicate level and sel_pte for each page caches, for better fanout 123 val req_info_dup = Vec(3, new L2TlbInnerBundle()) 124 val level_dup = Vec(3, UInt(log2Up(Level).W)) 125 val sel_pte_dup = Vec(3, UInt(XLEN.W)) 126 })) 127 val sfence_dup = Vec(4, Input(new SfenceBundle())) 128 val csr_dup = Vec(3, Input(new TlbCsrBundle())) 129} 130 131@chiselName 132class PtwCache()(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 133 val io = IO(new PtwCacheIO) 134 135 val ecc = Code.fromString(l2tlbParams.ecc) 136 val l2EntryType = new PTWEntriesWithEcc(ecc, num = PtwL2SectorSize, tagLen = PtwL2TagLen, level = 1, hasPerm = false) 137 val l3EntryType = new PTWEntriesWithEcc(ecc, num = PtwL3SectorSize, tagLen = PtwL3TagLen, level = 2, hasPerm = true) 138 139 // TODO: four caches make the codes dirty, think about how to deal with it 140 141 val sfence_dup = io.sfence_dup 142 val refill = io.refill.bits 143 val refill_prefetch_dup = io.refill.bits.req_info_dup.map(a => from_pre(a.source)) 144 val flush_dup = sfence_dup.zip(io.csr_dup).map(f => f._1.valid || f._2.satp.changed) 145 val flush = flush_dup(0) 146 147 // when refill, refuce to accept new req 148 val rwHarzad = if (sramSinglePort) io.refill.valid else false.B 149 150 // handle hand signal and req_info 151 // TODO: replace with FlushableQueue 152 val stageReq = Wire(Decoupled(new PtwCacheReq())) // enq stage & read page cache valid 153 val stageDelay = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // page cache resp 154 val stageCheck = Wire(Vec(2, Decoupled(new PtwCacheReq()))) // check hit & check ecc 155 val stageResp = Wire(Decoupled(new PtwCacheReq())) // deq stage 156 157 val stageDelay_valid_1cycle = OneCycleValid(stageReq.fire, flush) // catch ram data 158 val stageCheck_valid_1cycle = OneCycleValid(stageDelay(1).fire, flush) // replace & perf counter 159 val stageResp_valid_1cycle_dup = Wire(Vec(2, Bool())) 160 stageResp_valid_1cycle_dup.map(_ := OneCycleValid(stageCheck(1).fire, flush)) // ecc flush 161 162 stageReq <> io.req 163 PipelineConnect(stageReq, stageDelay(0), stageDelay(1).ready, flush, rwHarzad) 164 InsideStageConnect(stageDelay(0), stageDelay(1), stageDelay_valid_1cycle) 165 PipelineConnect(stageDelay(1), stageCheck(0), stageCheck(1).ready, flush) 166 InsideStageConnect(stageCheck(0), stageCheck(1), stageCheck_valid_1cycle) 167 PipelineConnect(stageCheck(1), stageResp, io.resp.ready, flush) 168 stageResp.ready := !stageResp.valid || io.resp.ready 169 170 // l1: level 0 non-leaf pte 171 val l1 = Reg(Vec(l2tlbParams.l1Size, new PtwEntry(tagLen = PtwL1TagLen))) 172 val l1v = RegInit(0.U(l2tlbParams.l1Size.W)) 173 val l1g = Reg(UInt(l2tlbParams.l1Size.W)) 174 val l1asids = l1.map(_.asid) 175 176 // l2: level 1 non-leaf pte 177 val l2 = Module(new SRAMTemplate( 178 l2EntryType, 179 set = l2tlbParams.l2nSets, 180 way = l2tlbParams.l2nWays, 181 singlePort = sramSinglePort 182 )) 183 val l2v = RegInit(0.U((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 184 val l2g = Reg(UInt((l2tlbParams.l2nSets * l2tlbParams.l2nWays).W)) 185 val l2asids = Reg(Vec(l2tlbParams.l2nSets, Vec(l2tlbParams.l2nWays, UInt(AsidLength.W)))) 186 def getl2vSet(vpn: UInt) = { 187 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 188 val set = genPtwL2SetIdx(vpn) 189 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 190 val l2vVec = l2v.asTypeOf(Vec(l2tlbParams.l2nSets, UInt(l2tlbParams.l2nWays.W))) 191 l2vVec(set) 192 } 193 def getl2asidSet(vpn: UInt) = { 194 require(log2Up(l2tlbParams.l2nWays) == log2Down(l2tlbParams.l2nWays)) 195 val set = genPtwL2SetIdx(vpn) 196 require(set.getWidth == log2Up(l2tlbParams.l2nSets)) 197 l2asids(set) 198 } 199 200 // l3: level 2 leaf pte of 4KB pages 201 val l3 = Module(new SRAMTemplate( 202 l3EntryType, 203 set = l2tlbParams.l3nSets, 204 way = l2tlbParams.l3nWays, 205 singlePort = sramSinglePort 206 )) 207 val l3v = RegInit(0.U((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 208 val l3g = Reg(UInt((l2tlbParams.l3nSets * l2tlbParams.l3nWays).W)) 209 val l3asids = Reg(Vec(l2tlbParams.l3nSets, Vec(l2tlbParams.l3nWays, UInt(AsidLength.W)))) 210 def getl3vSet(vpn: UInt) = { 211 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 212 val set = genPtwL3SetIdx(vpn) 213 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 214 val l3vVec = l3v.asTypeOf(Vec(l2tlbParams.l3nSets, UInt(l2tlbParams.l3nWays.W))) 215 l3vVec(set) 216 } 217 def getl3asidSet(vpn: UInt) = { 218 require(log2Up(l2tlbParams.l3nWays) == log2Down(l2tlbParams.l3nWays)) 219 val set = genPtwL3SetIdx(vpn) 220 require(set.getWidth == log2Up(l2tlbParams.l3nSets)) 221 l3asids(set) 222 } 223 224 // sp: level 0/1 leaf pte of 1GB/2MB super pages 225 val sp = Reg(Vec(l2tlbParams.spSize, new PtwEntry(tagLen = SPTagLen, hasPerm = true, hasLevel = true))) 226 val spv = RegInit(0.U(l2tlbParams.spSize.W)) 227 val spg = Reg(UInt(l2tlbParams.spSize.W)) 228 val spasids = sp.map(_.asid) 229 230 // Access Perf 231 val l1AccessPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 232 val l2AccessPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 233 val l3AccessPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 234 val spAccessPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 235 l1AccessPerf.map(_ := false.B) 236 l2AccessPerf.map(_ := false.B) 237 l3AccessPerf.map(_ := false.B) 238 spAccessPerf.map(_ := false.B) 239 240 241 242 def vpn_match(vpn1: UInt, vpn2: UInt, level: Int) = { 243 vpn1(vpnnLen*3-1, vpnnLen*(2-level)+3) === vpn2(vpnnLen*3-1, vpnnLen*(2-level)+3) 244 } 245 // NOTE: not actually bypassed, just check if hit, re-access the page cache 246 def refill_bypass(vpn: UInt, level: Int) = { 247 io.refill.valid && (level.U === io.refill.bits.level_dup(0)) && vpn_match(io.refill.bits.req_info_dup(0).vpn, vpn, level), 248 } 249 250 // l1 251 val ptwl1replace = ReplacementPolicy.fromString(l2tlbParams.l1Replacer, l2tlbParams.l1Size) 252 val (l1Hit, l1HitPPN, l1Pre) = { 253 val hitVecT = l1.zipWithIndex.map { case (e, i) => e.hit(stageReq.bits.req_info.vpn, io.csr_dup(0).satp.asid) && l1v(i) } 254 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 255 256 // stageDelay, but check for l1 257 val hitPPN = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.ppn)), stageDelay_valid_1cycle) 258 val hitPre = DataHoldBypass(ParallelPriorityMux(hitVec zip l1.map(_.prefetch)), stageDelay_valid_1cycle) 259 val hit = DataHoldBypass(ParallelOR(hitVec), stageDelay_valid_1cycle) 260 261 when (hit && stageDelay_valid_1cycle) { ptwl1replace.access(OHToUInt(hitVec)) } 262 263 l1AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageDelay_valid_1cycle} 264 for (i <- 0 until l2tlbParams.l1Size) { 265 XSDebug(stageReq.fire, p"[l1] l1(${i.U}) ${l1(i)} hit:${l1(i).hit(stageReq.bits.req_info.vpn, io.csr_dup(0).satp.asid)}\n") 266 } 267 XSDebug(stageReq.fire, p"[l1] l1v:${Binary(l1v)} hitVecT:${Binary(VecInit(hitVecT).asUInt)}\n") 268 XSDebug(stageDelay(0).valid, p"[l1] l1Hit:${hit} l1HitPPN:0x${Hexadecimal(hitPPN)} hitVec:${VecInit(hitVec).asUInt}\n") 269 270 VecInit(hitVecT).suggestName(s"l1_hitVecT") 271 VecInit(hitVec).suggestName(s"l1_hitVec") 272 273 // synchronize with other entries with RegEnable 274 (RegEnable(hit, stageDelay(1).fire), 275 RegEnable(hitPPN, stageDelay(1).fire), 276 RegEnable(hitPre, stageDelay(1).fire)) 277 } 278 279 // l2 280 val ptwl2replace = ReplacementPolicy.fromString(l2tlbParams.l2Replacer,l2tlbParams.l2nWays,l2tlbParams.l2nSets) 281 val (l2Hit, l2HitPPN, l2Pre, l2eccError) = { 282 val ridx = genPtwL2SetIdx(stageReq.bits.req_info.vpn) 283 l2.io.r.req.valid := stageReq.fire 284 l2.io.r.req.bits.apply(setIdx = ridx) 285 val vVec_req = getl2vSet(stageReq.bits.req_info.vpn) 286 287 // delay one cycle after sram read 288 val delay_vpn = stageDelay(0).bits.req_info.vpn 289 val data_resp = DataHoldBypass(l2.io.r.resp.data, stageDelay_valid_1cycle) 290 val vVec_delay = RegEnable(vVec_req, stageReq.fire) 291 val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).map { case (wayData, v) => 292 wayData.entries.hit(delay_vpn, io.csr_dup(1).satp.asid) && v }) 293 294 // check hit and ecc 295 val check_vpn = stageCheck(0).bits.req_info.vpn 296 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 297 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools() 298 299 val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire) 300 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 301 val hitWayData = hitWayEntry.entries 302 val hit = ParallelOR(hitVec) 303 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l2nWays).map(_.U(log2Up(l2tlbParams.l2nWays).W))) 304 val eccError = hitWayEntry.decode() 305 306 ridx.suggestName(s"l2_ridx") 307 ramDatas.suggestName(s"l2_ramDatas") 308 hitVec.suggestName(s"l2_hitVec") 309 hitWayData.suggestName(s"l2_hitWayData") 310 hitWay.suggestName(s"l2_hitWay") 311 312 when (hit && stageCheck_valid_1cycle) { ptwl2replace.access(genPtwL2SetIdx(check_vpn), hitWay) } 313 314 l2AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 315 XSDebug(stageDelay_valid_1cycle, p"[l2] ridx:0x${Hexadecimal(ridx)}\n") 316 for (i <- 0 until l2tlbParams.l2nWays) { 317 XSDebug(stageCheck_valid_1cycle, p"[l2] ramDatas(${i.U}) ${ramDatas(i)} l2v:${vVec(i)} hit:${hit}\n") 318 } 319 XSDebug(stageCheck_valid_1cycle, p"[l2] l2Hit:${hit} l2HitPPN:0x${Hexadecimal(hitWayData.ppns(genPtwL2SectorIdx(check_vpn)))} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} vidx:${vVec}\n") 320 321 (hit, hitWayData.ppns(genPtwL2SectorIdx(check_vpn)), hitWayData.prefetch, eccError) 322 } 323 324 // l3 325 val ptwl3replace = ReplacementPolicy.fromString(l2tlbParams.l3Replacer,l2tlbParams.l3nWays,l2tlbParams.l3nSets) 326 val (l3Hit, l3HitData, l3Pre, l3eccError) = { 327 val ridx = genPtwL3SetIdx(stageReq.bits.req_info.vpn) 328 l3.io.r.req.valid := stageReq.fire 329 l3.io.r.req.bits.apply(setIdx = ridx) 330 val vVec_req = getl3vSet(stageReq.bits.req_info.vpn) 331 332 // delay one cycle after sram read 333 val delay_vpn = stageDelay(0).bits.req_info.vpn 334 val data_resp = DataHoldBypass(l3.io.r.resp.data, stageDelay_valid_1cycle) 335 val vVec_delay = RegEnable(vVec_req, stageReq.fire) 336 val hitVec_delay = VecInit(data_resp.zip(vVec_delay.asBools).map { case (wayData, v) => 337 wayData.entries.hit(delay_vpn, io.csr_dup(2).satp.asid) && v }) 338 339 // check hit and ecc 340 val check_vpn = stageCheck(0).bits.req_info.vpn 341 val ramDatas = RegEnable(data_resp, stageDelay(1).fire) 342 val vVec = RegEnable(vVec_delay, stageDelay(1).fire).asBools() 343 344 val hitVec = RegEnable(hitVec_delay, stageDelay(1).fire) 345 val hitWayEntry = ParallelPriorityMux(hitVec zip ramDatas) 346 val hitWayData = hitWayEntry.entries 347 val hitWayEcc = hitWayEntry.ecc 348 val hit = ParallelOR(hitVec) 349 val hitWay = ParallelPriorityMux(hitVec zip (0 until l2tlbParams.l3nWays).map(_.U(log2Up(l2tlbParams.l3nWays).W))) 350 val eccError = hitWayEntry.decode() 351 352 when (hit && stageCheck_valid_1cycle) { ptwl3replace.access(genPtwL3SetIdx(check_vpn), hitWay) } 353 354 l3AccessPerf.zip(hitVec).map{ case (l, h) => l := h && stageCheck_valid_1cycle } 355 XSDebug(stageReq.fire, p"[l3] ridx:0x${Hexadecimal(ridx)}\n") 356 for (i <- 0 until l2tlbParams.l3nWays) { 357 XSDebug(stageCheck_valid_1cycle, p"[l3] ramDatas(${i.U}) ${ramDatas(i)} l3v:${vVec(i)} hit:${hitVec(i)}\n") 358 } 359 XSDebug(stageCheck_valid_1cycle, p"[l3] l3Hit:${hit} l3HitData:${hitWayData} hitVec:${Binary(hitVec.asUInt)} hitWay:${hitWay} v:${vVec}\n") 360 361 ridx.suggestName(s"l3_ridx") 362 ramDatas.suggestName(s"l3_ramDatas") 363 hitVec.suggestName(s"l3_hitVec") 364 hitWay.suggestName(s"l3_hitWay") 365 366 (hit, hitWayData, hitWayData.prefetch, eccError) 367 } 368 val l3HitPPN = l3HitData.ppns 369 val l3HitPerm = l3HitData.perms.getOrElse(0.U.asTypeOf(Vec(PtwL3SectorSize, new PtePermBundle))) 370 val l3HitValid = l3HitData.vs 371 372 // super page 373 val spreplace = ReplacementPolicy.fromString(l2tlbParams.spReplacer, l2tlbParams.spSize) 374 val (spHit, spHitData, spPre, spValid) = { 375 val hitVecT = sp.zipWithIndex.map { case (e, i) => e.hit(stageReq.bits.req_info.vpn, io.csr_dup(0).satp.asid) && spv(i) } 376 val hitVec = hitVecT.map(RegEnable(_, stageReq.fire)) 377 val hitData = ParallelPriorityMux(hitVec zip sp) 378 val hit = ParallelOR(hitVec) 379 380 when (hit && stageDelay_valid_1cycle) { spreplace.access(OHToUInt(hitVec)) } 381 382 spAccessPerf.zip(hitVec).map{ case (s, h) => s := h && stageDelay_valid_1cycle } 383 for (i <- 0 until l2tlbParams.spSize) { 384 XSDebug(stageReq.fire, p"[sp] sp(${i.U}) ${sp(i)} hit:${sp(i).hit(stageReq.bits.req_info.vpn, io.csr_dup(0).satp.asid)} spv:${spv(i)}\n") 385 } 386 XSDebug(stageDelay_valid_1cycle, p"[sp] spHit:${hit} spHitData:${hitData} hitVec:${Binary(VecInit(hitVec).asUInt)}\n") 387 388 VecInit(hitVecT).suggestName(s"sp_hitVecT") 389 VecInit(hitVec).suggestName(s"sp_hitVec") 390 391 (RegEnable(hit, stageDelay(1).fire), 392 RegEnable(hitData, stageDelay(1).fire), 393 RegEnable(hitData.prefetch, stageDelay(1).fire), 394 RegEnable(hitData.v, stageDelay(1).fire())) 395 } 396 val spHitPerm = spHitData.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)) 397 val spHitLevel = spHitData.level.getOrElse(0.U) 398 399 val check_res = Wire(new PageCacheRespBundle) 400 check_res.l1.apply(l1Hit, l1Pre, l1HitPPN) 401 check_res.l2.apply(l2Hit, l2Pre, l2HitPPN, ecc = l2eccError) 402 check_res.l3.apply(l3Hit, l3Pre, l3HitPPN, l3HitPerm, l3eccError, valid = l3HitValid) 403 check_res.sp.apply(spHit, spPre, spHitData.ppn, spHitPerm, false.B, spHitLevel, spValid) 404 405 val resp_res = Reg(new PageCacheRespBundle) 406 when (stageCheck(1).fire) { resp_res := check_res } 407 408 // stageResp bypass 409 val bypassed = Wire(Vec(3, Bool())) 410 bypassed.indices.foreach(i => 411 bypassed(i) := stageResp.bits.bypassed(i) || 412 ValidHoldBypass(refill_bypass(stageResp.bits.req_info.vpn, i), 413 OneCycleValid(stageCheck(1).fire, false.B) || io.refill.valid) 414 ) 415 416 io.resp.bits.req_info := stageResp.bits.req_info 417 io.resp.bits.isFirst := stageResp.bits.isFirst 418 io.resp.bits.hit := resp_res.l3.hit || resp_res.sp.hit 419 io.resp.bits.bypassed := bypassed(2) || (bypassed(1) && !resp_res.l2.hit) || (bypassed(0) && !resp_res.l1.hit) 420 io.resp.bits.prefetch := resp_res.l3.pre && resp_res.l3.hit || resp_res.sp.pre && resp_res.sp.hit 421 io.resp.bits.toFsm.l1Hit := resp_res.l1.hit 422 io.resp.bits.toFsm.l2Hit := resp_res.l2.hit 423 io.resp.bits.toFsm.ppn := Mux(resp_res.l2.hit, resp_res.l2.ppn, resp_res.l1.ppn) 424 io.resp.bits.toTlb.entry.map(_.tag := stageResp.bits.req_info.vpn(vpnLen - 1, 3)) 425 io.resp.bits.toTlb.entry.map(_.asid := io.csr_dup(0).satp.asid) // DontCare 426 io.resp.bits.toTlb.entry.map(_.level.map(_ := Mux(resp_res.l3.hit, 2.U, resp_res.sp.level))) 427 io.resp.bits.toTlb.entry.map(_.prefetch := from_pre(stageResp.bits.req_info.source)) 428 for (i <- 0 until tlbcontiguous) { 429 io.resp.bits.toTlb.entry(i).ppn := Mux(resp_res.l3.hit, resp_res.l3.ppn(i)(ppnLen - 1, sectortlbwidth), resp_res.sp.ppn(ppnLen - 1, sectortlbwidth)) 430 io.resp.bits.toTlb.entry(i).ppn_low := Mux(resp_res.l3.hit, resp_res.l3.ppn(i)(sectortlbwidth - 1, 0), resp_res.sp.ppn(sectortlbwidth - 1, 0)) 431 io.resp.bits.toTlb.entry(i).perm.map(_ := Mux(resp_res.l3.hit, resp_res.l3.perm(i), resp_res.sp.perm)) 432 io.resp.bits.toTlb.entry(i).v := Mux(resp_res.l3.hit, resp_res.l3.v(i), resp_res.sp.v) 433 io.resp.bits.toTlb.entry(i).pf := !io.resp.bits.toTlb.entry(i).v 434 io.resp.bits.toTlb.entry(i).af := false.B 435 } 436 io.resp.bits.toTlb.pteidx := UIntToOH(stageResp.bits.req_info.vpn(2, 0)).asBools 437 io.resp.bits.toTlb.not_super := Mux(resp_res.l3.hit, true.B, false.B) 438 io.resp.valid := stageResp.valid 439 XSError(stageResp.valid && resp_res.l3.hit && resp_res.sp.hit, "normal page and super page both hit") 440 XSError(stageResp.valid && io.resp.bits.hit && bypassed(2), "page cache, bypassed but hit") 441 442 // refill Perf 443 val l1RefillPerf = Wire(Vec(l2tlbParams.l1Size, Bool())) 444 val l2RefillPerf = Wire(Vec(l2tlbParams.l2nWays, Bool())) 445 val l3RefillPerf = Wire(Vec(l2tlbParams.l3nWays, Bool())) 446 val spRefillPerf = Wire(Vec(l2tlbParams.spSize, Bool())) 447 l1RefillPerf.map(_ := false.B) 448 l2RefillPerf.map(_ := false.B) 449 l3RefillPerf.map(_ := false.B) 450 spRefillPerf.map(_ := false.B) 451 452 // refill 453 l2.io.w.req <> DontCare 454 l3.io.w.req <> DontCare 455 l2.io.w.req.valid := false.B 456 l3.io.w.req.valid := false.B 457 458 val memRdata = refill.ptes 459 val memPtes = (0 until (l2tlbParams.blockBytes/(XLEN/8))).map(i => memRdata((i+1)*XLEN-1, i*XLEN).asTypeOf(new PteBundle)) 460 val memSelData = io.refill.bits.sel_pte_dup 461 val memPte = memSelData.map(a => a.asTypeOf(new PteBundle)) 462 463 // TODO: handle sfenceLatch outsize 464 when (!flush_dup(0) && refill.levelOH.l1 && !memPte(0).isLeaf() && !memPte(0).isPf(refill.level_dup(0)) && !memPte(0).isAf()) { 465 // val refillIdx = LFSR64()(log2Up(l2tlbParams.l1Size)-1,0) // TODO: may be LRU 466 val refillIdx = replaceWrapper(l1v, ptwl1replace.way) 467 refillIdx.suggestName(s"PtwL1RefillIdx") 468 val rfOH = UIntToOH(refillIdx) 469 l1(refillIdx).refill( 470 refill.req_info_dup(0).vpn, 471 io.csr_dup(0).satp.asid, 472 memSelData(0), 473 0.U, 474 refill_prefetch_dup(0) 475 ) 476 ptwl1replace.access(refillIdx) 477 l1v := l1v | rfOH 478 l1g := (l1g & ~rfOH) | Mux(memPte(0).perm.g, rfOH, 0.U) 479 480 for (i <- 0 until l2tlbParams.l1Size) { 481 l1RefillPerf(i) := i.U === refillIdx 482 } 483 484 XSDebug(p"[l1 refill] refillIdx:${refillIdx} refillEntry:${l1(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, io.csr_dup(0).satp.asid, memSelData(0), 0.U, prefetch = refill_prefetch_dup(0))}\n") 485 XSDebug(p"[l1 refill] l1v:${Binary(l1v)}->${Binary(l1v | rfOH)} l1g:${Binary(l1g)}->${Binary((l1g & ~rfOH) | Mux(memPte(0).perm.g, rfOH, 0.U))}\n") 486 487 refillIdx.suggestName(s"l1_refillIdx") 488 rfOH.suggestName(s"l1_rfOH") 489 } 490 491 when (!flush_dup(1) && refill.levelOH.l2 && !memPte(1).isLeaf() && !memPte(1).isPf(refill.level_dup(1)) && !memPte(1).isAf()) { 492 val refillIdx = genPtwL2SetIdx(refill.req_info_dup(1).vpn) 493 val victimWay = replaceWrapper(getl2vSet(refill.req_info_dup(1).vpn), ptwl2replace.way(refillIdx)) 494 val victimWayOH = UIntToOH(victimWay) 495 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 496 val wdata = Wire(l2EntryType) 497 wdata.gen( 498 vpn = refill.req_info_dup(1).vpn, 499 asid = io.csr_dup(1).satp.asid, 500 data = memRdata, 501 levelUInt = 1.U, 502 refill_prefetch_dup(1) 503 ) 504 l2.io.w.apply( 505 valid = true.B, 506 setIdx = refillIdx, 507 data = wdata, 508 waymask = victimWayOH 509 ) 510 ptwl2replace.access(refillIdx, victimWay) 511 l2v := l2v | rfvOH 512 l2g := l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 513 514 for (i <- 0 until l2tlbParams.l2nWays) { 515 l2RefillPerf(i) := i.U === victimWay 516 } 517 518 XSDebug(p"[l2 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 519 XSDebug(p"[l2 refill] refilldata:0x${wdata}\n") 520 XSDebug(p"[l2 refill] l2v:${Binary(l2v)} -> ${Binary(l2v | rfvOH)}\n") 521 XSDebug(p"[l2 refill] l2g:${Binary(l2g)} -> ${Binary(l2g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 522 523 refillIdx.suggestName(s"l2_refillIdx") 524 victimWay.suggestName(s"l2_victimWay") 525 victimWayOH.suggestName(s"l2_victimWayOH") 526 rfvOH.suggestName(s"l2_rfvOH") 527 } 528 529 when (!flush_dup(2) && refill.levelOH.l3 && !memPte(2).isAf()) { 530 val refillIdx = genPtwL3SetIdx(refill.req_info_dup(2).vpn) 531 val victimWay = replaceWrapper(getl3vSet(refill.req_info_dup(2).vpn), ptwl3replace.way(refillIdx)) 532 val victimWayOH = UIntToOH(victimWay) 533 val rfvOH = UIntToOH(Cat(refillIdx, victimWay)) 534 val wdata = Wire(l3EntryType) 535 wdata.gen( 536 vpn = refill.req_info_dup(2).vpn, 537 asid = io.csr_dup(2).satp.asid, 538 data = memRdata, 539 levelUInt = 2.U, 540 refill_prefetch_dup(2) 541 ) 542 l3.io.w.apply( 543 valid = true.B, 544 setIdx = refillIdx, 545 data = wdata, 546 waymask = victimWayOH 547 ) 548 ptwl3replace.access(refillIdx, victimWay) 549 l3v := l3v | rfvOH 550 l3g := l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U) 551 552 for (i <- 0 until l2tlbParams.l3nWays) { 553 l3RefillPerf(i) := i.U === victimWay 554 } 555 556 XSDebug(p"[l3 refill] refillIdx:0x${Hexadecimal(refillIdx)} victimWay:${victimWay} victimWayOH:${Binary(victimWayOH)} rfvOH(in UInt):${Cat(refillIdx, victimWay)}\n") 557 XSDebug(p"[l3 refill] refilldata:0x${wdata}\n") 558 XSDebug(p"[l3 refill] l3v:${Binary(l3v)} -> ${Binary(l3v | rfvOH)}\n") 559 XSDebug(p"[l3 refill] l3g:${Binary(l3g)} -> ${Binary(l3g & ~rfvOH | Mux(Cat(memPtes.map(_.perm.g)).andR, rfvOH, 0.U))}\n") 560 561 refillIdx.suggestName(s"l3_refillIdx") 562 victimWay.suggestName(s"l3_victimWay") 563 victimWayOH.suggestName(s"l3_victimWayOH") 564 rfvOH.suggestName(s"l3_rfvOH") 565 } 566 567 568 // misc entries: super & invalid 569 when (!flush_dup(0) && refill.levelOH.sp && (memPte(0).isLeaf() || memPte(0).isPf(refill.level_dup(0))) && !memPte(0).isAf()) { 570 val refillIdx = spreplace.way// LFSR64()(log2Up(l2tlbParams.spSize)-1,0) // TODO: may be LRU 571 val rfOH = UIntToOH(refillIdx) 572 sp(refillIdx).refill( 573 refill.req_info_dup(0).vpn, 574 io.csr_dup(0).satp.asid, 575 memSelData(0), 576 refill.level_dup(2), 577 refill_prefetch_dup(0), 578 !memPte(0).isPf(refill.level_dup(0)), 579 ) 580 spreplace.access(refillIdx) 581 spv := spv | rfOH 582 spg := spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U) 583 584 for (i <- 0 until l2tlbParams.spSize) { 585 spRefillPerf(i) := i.U === refillIdx 586 } 587 588 XSDebug(p"[sp refill] refillIdx:${refillIdx} refillEntry:${sp(refillIdx).genPtwEntry(refill.req_info_dup(0).vpn, io.csr_dup(0).satp.asid, memSelData(0), refill.level_dup(0), refill_prefetch_dup(0))}\n") 589 XSDebug(p"[sp refill] spv:${Binary(spv)}->${Binary(spv | rfOH)} spg:${Binary(spg)}->${Binary(spg & ~rfOH | Mux(memPte(0).perm.g, rfOH, 0.U))}\n") 590 591 refillIdx.suggestName(s"sp_refillIdx") 592 rfOH.suggestName(s"sp_rfOH") 593 } 594 595 val l2eccFlush = resp_res.l2.ecc && stageResp_valid_1cycle_dup(0) // RegNext(l2eccError, init = false.B) 596 val l3eccFlush = resp_res.l3.ecc && stageResp_valid_1cycle_dup(1) // RegNext(l3eccError, init = false.B) 597 val eccVpn = stageResp.bits.req_info.vpn 598 599 XSError(l2eccFlush, "l2tlb.cache.l2 ecc error. Should not happen at sim stage") 600 XSError(l3eccFlush, "l2tlb.cache.l3 ecc error. Should not happen at sim stage") 601 when (l2eccFlush) { 602 val flushSetIdxOH = UIntToOH(genPtwL2SetIdx(eccVpn)) 603 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l2nWays, a.asUInt) }).asUInt 604 l2v := l2v & ~flushMask 605 l2g := l2g & ~flushMask 606 } 607 608 when (l3eccFlush) { 609 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(eccVpn)) 610 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 611 l3v := l3v & ~flushMask 612 l3g := l3g & ~flushMask 613 } 614 615 // sfence 616 when (sfence_dup(3).valid) { 617 val sfence_vpn = sfence_dup(3).bits.addr(sfence_dup(3).bits.addr.getWidth-1, offLen) 618 619 when (sfence_dup(3).bits.rs1/*va*/) { 620 when (sfence_dup(3).bits.rs2) { 621 // all va && all asid 622 l3v := 0.U 623 } .otherwise { 624 // all va && specific asid except global 625 l3v := l3v & l3g 626 } 627 } .otherwise { 628 // val flushMask = UIntToOH(genTlbL2Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen))) 629 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(sfence_vpn)) 630 // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l3nWays, _.asUInt))).asUInt 631 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 632 flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH") 633 flushMask.suggestName(s"sfence_nrs1_flushMask") 634 635 when (sfence_dup(3).bits.rs2) { 636 // specific leaf of addr && all asid 637 l3v := l3v & ~flushMask 638 } .otherwise { 639 // specific leaf of addr && specific asid 640 l3v := l3v & (~flushMask | l3g) 641 } 642 } 643 } 644 645 when (sfence_dup(0).valid) { 646 val l1asidhit = VecInit(l1asids.map(_ === sfence_dup(0).bits.asid)).asUInt 647 val spasidhit = VecInit(spasids.map(_ === sfence_dup(0).bits.asid)).asUInt 648 val sfence_vpn = sfence_dup(0).bits.addr(sfence_dup(0).bits.addr.getWidth-1, offLen) 649 650 when (sfence_dup(0).bits.rs1/*va*/) { 651 when (sfence_dup(0).bits.rs2) { 652 // all va && all asid 653 l1v := 0.U 654 l2v := 0.U 655 spv := 0.U 656 } .otherwise { 657 // all va && specific asid except global 658 659 l1v := l1v & (~l1asidhit | l1g) 660 l2v := l2v & l2g 661 spv := spv & (~spasidhit | spg) 662 } 663 } .otherwise { 664 // val flushMask = UIntToOH(genTlbL2Idx(sfence.bits.addr(sfence.bits.addr.getWidth-1, offLen))) 665 val flushSetIdxOH = UIntToOH(genPtwL3SetIdx(sfence_vpn)) 666 // val flushMask = VecInit(flushSetIdxOH.asBools.map(Fill(l2tlbParams.l3nWays, _.asUInt))).asUInt 667 val flushMask = VecInit(flushSetIdxOH.asBools.map { a => Fill(l2tlbParams.l3nWays, a.asUInt) }).asUInt 668 flushSetIdxOH.suggestName(s"sfence_nrs1_flushSetIdxOH") 669 flushMask.suggestName(s"sfence_nrs1_flushMask") 670 671 when (sfence_dup(0).bits.rs2) { 672 // specific leaf of addr && all asid 673 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.asid, ignoreAsid = true))).asUInt) 674 } .otherwise { 675 // specific leaf of addr && specific asid 676 spv := spv & (~VecInit(sp.map(_.hit(sfence_vpn, sfence_dup(0).bits.asid))).asUInt | spg) 677 } 678 } 679 } 680 681 def InsideStageConnect(in: DecoupledIO[PtwCacheReq], out: DecoupledIO[PtwCacheReq], inFire: Bool): Unit = { 682 in.ready := !in.valid || out.ready 683 out.valid := in.valid 684 out.bits := in.bits 685 out.bits.bypassed.zip(in.bits.bypassed).zipWithIndex.map{ case (b, i) => 686 val bypassed_reg = Reg(Bool()) 687 val bypassed_wire = refill_bypass(in.bits.req_info.vpn, i) && io.refill.valid 688 when (inFire) { bypassed_reg := bypassed_wire } 689 .elsewhen (io.refill.valid) { bypassed_reg := bypassed_reg || bypassed_wire } 690 691 b._1 := b._2 || (bypassed_wire || (bypassed_reg && !inFire)) 692 } 693 } 694 695 // Perf Count 696 val resp_l3 = resp_res.l3.hit 697 val resp_sp = resp_res.sp.hit 698 val resp_l1_pre = resp_res.l1.pre 699 val resp_l2_pre = resp_res.l2.pre 700 val resp_l3_pre = resp_res.l3.pre 701 val resp_sp_pre = resp_res.sp.pre 702 val base_valid_access_0 = !from_pre(io.resp.bits.req_info.source) && io.resp.fire() 703 XSPerfAccumulate("access", base_valid_access_0) 704 XSPerfAccumulate("l1_hit", base_valid_access_0 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 705 XSPerfAccumulate("l2_hit", base_valid_access_0 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 706 XSPerfAccumulate("l3_hit", base_valid_access_0 && resp_l3) 707 XSPerfAccumulate("sp_hit", base_valid_access_0 && resp_sp) 708 XSPerfAccumulate("pte_hit",base_valid_access_0 && io.resp.bits.hit) 709 710 XSPerfAccumulate("l1_hit_pre", base_valid_access_0 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 711 XSPerfAccumulate("l2_hit_pre", base_valid_access_0 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 712 XSPerfAccumulate("l3_hit_pre", base_valid_access_0 && resp_l3_pre && resp_l3) 713 XSPerfAccumulate("sp_hit_pre", base_valid_access_0 && resp_sp_pre && resp_sp) 714 XSPerfAccumulate("pte_hit_pre",base_valid_access_0 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 715 716 val base_valid_access_1 = from_pre(io.resp.bits.req_info.source) && io.resp.fire() 717 XSPerfAccumulate("pre_access", base_valid_access_1) 718 XSPerfAccumulate("pre_l1_hit", base_valid_access_1 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 719 XSPerfAccumulate("pre_l2_hit", base_valid_access_1 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 720 XSPerfAccumulate("pre_l3_hit", base_valid_access_1 && resp_l3) 721 XSPerfAccumulate("pre_sp_hit", base_valid_access_1 && resp_sp) 722 XSPerfAccumulate("pre_pte_hit",base_valid_access_1 && io.resp.bits.hit) 723 724 XSPerfAccumulate("pre_l1_hit_pre", base_valid_access_1 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 725 XSPerfAccumulate("pre_l2_hit_pre", base_valid_access_1 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 726 XSPerfAccumulate("pre_l3_hit_pre", base_valid_access_1 && resp_l3_pre && resp_l3) 727 XSPerfAccumulate("pre_sp_hit_pre", base_valid_access_1 && resp_sp_pre && resp_sp) 728 XSPerfAccumulate("pre_pte_hit_pre",base_valid_access_1 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 729 730 val base_valid_access_2 = stageResp.bits.isFirst && !from_pre(io.resp.bits.req_info.source) && io.resp.fire() 731 XSPerfAccumulate("access_first", base_valid_access_2) 732 XSPerfAccumulate("l1_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 733 XSPerfAccumulate("l2_hit_first", base_valid_access_2 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 734 XSPerfAccumulate("l3_hit_first", base_valid_access_2 && resp_l3) 735 XSPerfAccumulate("sp_hit_first", base_valid_access_2 && resp_sp) 736 XSPerfAccumulate("pte_hit_first",base_valid_access_2 && io.resp.bits.hit) 737 738 XSPerfAccumulate("l1_hit_pre_first", base_valid_access_2 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 739 XSPerfAccumulate("l2_hit_pre_first", base_valid_access_2 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 740 XSPerfAccumulate("l3_hit_pre_first", base_valid_access_2 && resp_l3_pre && resp_l3) 741 XSPerfAccumulate("sp_hit_pre_first", base_valid_access_2 && resp_sp_pre && resp_sp) 742 XSPerfAccumulate("pte_hit_pre_first",base_valid_access_2 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 743 744 val base_valid_access_3 = stageResp.bits.isFirst && from_pre(io.resp.bits.req_info.source) && io.resp.fire() 745 XSPerfAccumulate("pre_access_first", base_valid_access_3) 746 XSPerfAccumulate("pre_l1_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 747 XSPerfAccumulate("pre_l2_hit_first", base_valid_access_3 && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 748 XSPerfAccumulate("pre_l3_hit_first", base_valid_access_3 && resp_l3) 749 XSPerfAccumulate("pre_sp_hit_first", base_valid_access_3 && resp_sp) 750 XSPerfAccumulate("pre_pte_hit_first", base_valid_access_3 && io.resp.bits.hit) 751 752 XSPerfAccumulate("pre_l1_hit_pre_first", base_valid_access_3 && resp_l1_pre && io.resp.bits.toFsm.l1Hit && !io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 753 XSPerfAccumulate("pre_l2_hit_pre_first", base_valid_access_3 && resp_l2_pre && io.resp.bits.toFsm.l2Hit && !io.resp.bits.hit) 754 XSPerfAccumulate("pre_l3_hit_pre_first", base_valid_access_3 && resp_l3_pre && resp_l3) 755 XSPerfAccumulate("pre_sp_hit_pre_first", base_valid_access_3 && resp_sp_pre && resp_sp) 756 XSPerfAccumulate("pre_pte_hit_pre_first",base_valid_access_3 && (resp_l3_pre && resp_l3 || resp_sp_pre && resp_sp) && io.resp.bits.hit) 757 758 XSPerfAccumulate("rwHarzad", io.req.valid && !io.req.ready) 759 XSPerfAccumulate("out_blocked", io.resp.valid && !io.resp.ready) 760 l1AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1AccessIndex${i}", l) } 761 l2AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2AccessIndex${i}", l) } 762 l3AccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3AccessIndex${i}", l) } 763 spAccessPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPAccessIndex${i}", l) } 764 l1RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L1RefillIndex${i}", l) } 765 l2RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L2RefillIndex${i}", l) } 766 l3RefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"L3RefillIndex${i}", l) } 767 spRefillPerf.zipWithIndex.map{ case (l, i) => XSPerfAccumulate(s"SPRefillIndex${i}", l) } 768 769 XSPerfAccumulate("l1Refill", Cat(l1RefillPerf).orR) 770 XSPerfAccumulate("l2Refill", Cat(l2RefillPerf).orR) 771 XSPerfAccumulate("l3Refill", Cat(l3RefillPerf).orR) 772 XSPerfAccumulate("spRefill", Cat(spRefillPerf).orR) 773 XSPerfAccumulate("l1Refill_pre", Cat(l1RefillPerf).orR && refill_prefetch_dup(0)) 774 XSPerfAccumulate("l2Refill_pre", Cat(l2RefillPerf).orR && refill_prefetch_dup(0)) 775 XSPerfAccumulate("l3Refill_pre", Cat(l3RefillPerf).orR && refill_prefetch_dup(0)) 776 XSPerfAccumulate("spRefill_pre", Cat(spRefillPerf).orR && refill_prefetch_dup(0)) 777 778 // debug 779 XSDebug(sfence_dup(0).valid, p"[sfence] original v and g vector:\n") 780 XSDebug(sfence_dup(0).valid, p"[sfence] l1v:${Binary(l1v)}\n") 781 XSDebug(sfence_dup(0).valid, p"[sfence] l2v:${Binary(l2v)}\n") 782 XSDebug(sfence_dup(0).valid, p"[sfence] l3v:${Binary(l3v)}\n") 783 XSDebug(sfence_dup(0).valid, p"[sfence] l3g:${Binary(l3g)}\n") 784 XSDebug(sfence_dup(0).valid, p"[sfence] spv:${Binary(spv)}\n") 785 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] new v and g vector:\n") 786 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l1v:${Binary(l1v)}\n") 787 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l2v:${Binary(l2v)}\n") 788 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3v:${Binary(l3v)}\n") 789 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] l3g:${Binary(l3g)}\n") 790 XSDebug(RegNext(sfence_dup(0).valid), p"[sfence] spv:${Binary(spv)}\n") 791 792 val perfEvents = Seq( 793 ("access ", base_valid_access_0 ), 794 ("l1_hit ", l1Hit ), 795 ("l2_hit ", l2Hit ), 796 ("l3_hit ", l3Hit ), 797 ("sp_hit ", spHit ), 798 ("pte_hit ", l3Hit || spHit ), 799 ("rwHarzad ", io.req.valid && !io.req.ready ), 800 ("out_blocked ", io.resp.valid && !io.resp.ready), 801 ) 802 generatePerfEvent() 803} 804