1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.internal.naming.chiselName 22import chisel3.util._ 23import freechips.rocketchip.util.SRAMAnnotation 24import xiangshan._ 25import utils._ 26import xiangshan.backend.fu.{PMPChecker, PMPReqBundle} 27import xiangshan.backend.rob.RobPtr 28import xiangshan.backend.fu.util.HasCSRConst 29 30 31@chiselName 32class TLB(Width: Int, q: TLBParameters)(implicit p: Parameters) extends TlbModule with HasCSRConst { 33 val io = IO(new TlbIO(Width, q)) 34 35 require(q.superAssociative == "fa") 36 if (q.sameCycle) { 37 require(q.normalAssociative == "fa") 38 } 39 40 val req = io.requestor.map(_.req) 41 val resp = io.requestor.map(_.resp) 42 val ptw = io.ptw 43 val pmp = io.pmp 44 45 val sfence = io.sfence 46 val csr = io.csr 47 val satp = csr.satp 48 val priv = csr.priv 49 val ifecth = if (q.fetchi) true.B else false.B 50 val mode = if (q.useDmode) priv.dmode else priv.imode 51 // val vmEnable = satp.mode === 8.U // && (mode < ModeM) // FIXME: fix me when boot xv6/linux... 52 val vmEnable = if (EnbaleTlbDebug) (satp.mode === 8.U) 53 else (satp.mode === 8.U && (mode < ModeM)) 54 55 val reqAddr = req.map(_.bits.vaddr.asTypeOf((new VaBundle).cloneType)) 56 val vpn = reqAddr.map(_.vpn) 57 val cmd = req.map(_.bits.cmd) 58 val valid = req.map(_.valid) 59 60 def widthMapSeq[T <: Seq[Data]](f: Int => T) = (0 until Width).map(f) 61 62 def widthMap[T <: Data](f: Int => T) = (0 until Width).map(f) 63 64 // Normal page && Super page 65 val normalPage = TlbStorage( 66 name = "normal", 67 associative = q.normalAssociative, 68 sameCycle = q.sameCycle, 69 ports = Width, 70 nSets = q.normalNSets, 71 nWays = q.normalNWays, 72 sramSinglePort = sramSinglePort, 73 normalPage = true, 74 superPage = false 75 ) 76 val superPage = TlbStorage( 77 name = "super", 78 associative = q.superAssociative, 79 sameCycle = q.sameCycle, 80 ports = Width, 81 nSets = q.superNSets, 82 nWays = q.superNWays, 83 sramSinglePort = sramSinglePort, 84 normalPage = q.normalAsVictim, 85 superPage = true, 86 ) 87 88 89 for (i <- 0 until Width) { 90 normalPage.r_req_apply( 91 valid = io.requestor(i).req.valid, 92 vpn = vpn(i), 93 asid = csr.satp.asid, 94 i = i 95 ) 96 superPage.r_req_apply( 97 valid = io.requestor(i).req.valid, 98 vpn = vpn(i), 99 asid = csr.satp.asid, 100 i = i 101 ) 102 } 103 104 normalPage.victim.in <> superPage.victim.out 105 normalPage.victim.out <> superPage.victim.in 106 normalPage.sfence <> io.sfence 107 superPage.sfence <> io.sfence 108 normalPage.csr <> io.csr 109 superPage.csr <> io.csr 110 111 def TLBNormalRead(i: Int) = { 112 val (normal_hit, normal_ppn, normal_perm) = normalPage.r_resp_apply(i) 113 val (super_hit, super_ppn, super_perm) = superPage.r_resp_apply(i) 114 assert(!(normal_hit && super_hit && vmEnable && RegNext(req(i).valid, init = false.B))) 115 116 val hit = normal_hit || super_hit 117 val ppn = Mux(normal_hit, normal_ppn, super_ppn) 118 val perm = Mux(normal_hit, normal_perm, super_perm) 119 120 val pf = perm.pf && hit 121 val af = perm.af && hit 122 val cmdReg = if (!q.sameCycle) RegNext(cmd(i)) else cmd(i) 123 val validReg = if (!q.sameCycle) RegNext(valid(i)) else valid(i) 124 val offReg = if (!q.sameCycle) RegNext(reqAddr(i).off) else reqAddr(i).off 125 val sizeReg = if (!q.sameCycle) RegNext(req(i).bits.size) else req(i).bits.size 126 127 /** *************** next cycle when two cycle is false******************* */ 128 val miss = !hit && vmEnable 129 hit.suggestName(s"hit_${i}") 130 miss.suggestName(s"miss_${i}") 131 132 XSDebug(validReg, p"(${i.U}) hit:${hit} miss:${miss} ppn:${Hexadecimal(ppn)} perm:${perm}\n") 133 134 val paddr = Cat(ppn, offReg) 135 val vaddr = SignExt(req(i).bits.vaddr, PAddrBits) 136 137 req(i).ready := resp(i).ready 138 resp(i).valid := validReg 139 resp(i).bits.paddr := Mux(vmEnable, paddr, if (!q.sameCycle) RegNext(vaddr) else vaddr) 140 resp(i).bits.miss := miss 141 resp(i).bits.ptwBack := io.ptw.resp.fire() 142 143 pmp(i).valid := resp(i).valid 144 pmp(i).bits.addr := resp(i).bits.paddr 145 pmp(i).bits.size := sizeReg 146 pmp(i).bits.cmd := cmdReg 147 148 val ldUpdate = hit && !perm.a && TlbCmd.isRead(cmdReg) && !TlbCmd.isAmo(cmdReg) // update A/D through exception 149 val stUpdate = hit && (!perm.a || !perm.d) && (TlbCmd.isWrite(cmdReg) || TlbCmd.isAmo(cmdReg)) // update A/D through exception 150 val instrUpdate = hit && !perm.a && TlbCmd.isExec(cmdReg) // update A/D through exception 151 val modeCheck = !(mode === ModeU && !perm.u || mode === ModeS && perm.u && (!priv.sum || ifecth)) 152 val ldPermFail = !(modeCheck && (perm.r || priv.mxr && perm.x)) 153 val stPermFail = !(modeCheck && perm.w) 154 val instrPermFail = !(modeCheck && perm.x) 155 val ldPf = (ldPermFail || pf) && (TlbCmd.isRead(cmdReg) && !TlbCmd.isAmo(cmdReg)) 156 val stPf = (stPermFail || pf) && (TlbCmd.isWrite(cmdReg) || TlbCmd.isAmo(cmdReg)) 157 val instrPf = (instrPermFail || pf) && TlbCmd.isExec(cmdReg) 158 resp(i).bits.excp.pf.ld := (ldPf || ldUpdate) && vmEnable && hit && !af 159 resp(i).bits.excp.pf.st := (stPf || stUpdate) && vmEnable && hit && !af 160 resp(i).bits.excp.pf.instr := (instrPf || instrUpdate) && vmEnable && hit && !af 161 // NOTE: pf need && with !af, page fault has higher priority than access fault 162 // but ptw may also have access fault, then af happens, the translation is wrong. 163 // In this case, pf has lower priority than af 164 165 // if vmenable, use pre-calcuated pma check result 166 resp(i).bits.mmio := Mux(TlbCmd.isExec(cmdReg), !perm.pi, !perm.pd) && vmEnable && hit 167 resp(i).bits.excp.af.ld := (af || Mux(TlbCmd.isAtom(cmdReg), !perm.pa, !perm.pr) && TlbCmd.isRead(cmdReg)) && vmEnable && hit 168 resp(i).bits.excp.af.st := (af || Mux(TlbCmd.isAtom(cmdReg), !perm.pa, !perm.pw) && TlbCmd.isWrite(cmdReg)) && vmEnable && hit 169 resp(i).bits.excp.af.instr := (af || Mux(TlbCmd.isAtom(cmdReg), false.B, !perm.pe)) && vmEnable && hit 170 171 // if !vmenable, check pma 172 val (pmaMode, accessWidth) = AddressSpace.memmapAddrMatch(resp(i).bits.paddr) 173 when(!vmEnable) { 174 resp(i).bits.mmio := Mux(TlbCmd.isExec(cmdReg), !PMAMode.icache(pmaMode), !PMAMode.dcache(pmaMode)) 175 resp(i).bits.excp.af.ld := Mux(TlbCmd.isAtom(cmdReg), !PMAMode.atomic(pmaMode), !PMAMode.read(pmaMode)) && TlbCmd.isRead(cmdReg) 176 resp(i).bits.excp.af.st := Mux(TlbCmd.isAtom(cmdReg), !PMAMode.atomic(pmaMode), !PMAMode.write(pmaMode)) && TlbCmd.isWrite(cmdReg) 177 resp(i).bits.excp.af.instr := Mux(TlbCmd.isAtom(cmdReg), false.B, !PMAMode.execute(pmaMode)) 178 } 179 180 (hit, miss, validReg) 181 } 182 183 val readResult = (0 until Width).map(TLBNormalRead(_)) 184 val hitVec = readResult.map(_._1) 185 val missVec = readResult.map(_._2) 186 val validRegVec = readResult.map(_._3) 187 188 // replacement 189 def get_access(one_hot: UInt, valid: Bool): Valid[UInt] = { 190 val res = Wire(Valid(UInt(log2Up(one_hot.getWidth).W))) 191 res.valid := Cat(one_hot).orR && valid 192 res.bits := OHToUInt(one_hot) 193 res 194 } 195 196 val normal_refill_idx = if (q.outReplace) { 197 io.replace.normalPage.access <> normalPage.access 198 io.replace.normalPage.chosen_set := get_set_idx(io.ptw.resp.bits.entry.tag, q.normalNSets) 199 io.replace.normalPage.refillIdx 200 } else if (q.normalAssociative == "fa") { 201 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNWays) 202 re.access(normalPage.access.map(_.touch_ways)) // normalhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, validRegVec(i))}) 203 re.way 204 } else { // set-acco && plru 205 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNSets, q.normalNWays) 206 re.access(normalPage.access.map(_.sets), normalPage.access.map(_.touch_ways)) 207 re.way(get_set_idx(io.ptw.resp.bits.entry.tag, q.normalNSets)) 208 } 209 210 val super_refill_idx = if (q.outReplace) { 211 io.replace.superPage.access <> superPage.access 212 io.replace.superPage.chosen_set := DontCare 213 io.replace.superPage.refillIdx 214 } else { 215 val re = ReplacementPolicy.fromString(q.superReplacer, q.superNWays) 216 re.access(superPage.access.map(_.touch_ways)) 217 re.way 218 } 219 220 val refill = ptw.resp.fire() && !sfence.valid && !satp.changed 221 normalPage.w_apply( 222 valid = { if (q.normalAsVictim) false.B 223 else refill && ptw.resp.bits.entry.level.get === 2.U }, 224 wayIdx = normal_refill_idx, 225 data = ptw.resp.bits 226 ) 227 superPage.w_apply( 228 valid = { if (q.normalAsVictim) refill 229 else refill && ptw.resp.bits.entry.level.get =/= 2.U }, 230 wayIdx = super_refill_idx, 231 data = ptw.resp.bits 232 ) 233 234 for (i <- 0 until Width) { 235 io.ptw.req(i).valid := validRegVec(i) && missVec(i) && !RegNext(refill) 236 io.ptw.req(i).bits.vpn := RegNext(reqAddr(i).vpn) 237 } 238 io.ptw.resp.ready := true.B 239 240 if (!q.shouldBlock) { 241 for (i <- 0 until Width) { 242 XSPerfAccumulate("first_access" + Integer.toString(i, 10), validRegVec(i) && vmEnable && RegNext(req(i).bits.debug.isFirstIssue)) 243 XSPerfAccumulate("access" + Integer.toString(i, 10), validRegVec(i) && vmEnable) 244 } 245 for (i <- 0 until Width) { 246 XSPerfAccumulate("first_miss" + Integer.toString(i, 10), validRegVec(i) && vmEnable && missVec(i) && RegNext(req(i).bits.debug.isFirstIssue)) 247 XSPerfAccumulate("miss" + Integer.toString(i, 10), validRegVec(i) && vmEnable && missVec(i)) 248 } 249 } else { 250 // NOTE: ITLB is blocked, so every resp will be valid only when hit 251 // every req will be ready only when hit 252 for (i <- 0 until Width) { 253 XSPerfAccumulate(s"access${i}", io.requestor(i).req.fire() && vmEnable) 254 XSPerfAccumulate(s"miss${i}", ptw.req(i).fire()) 255 } 256 257 } 258 //val reqCycleCnt = Reg(UInt(16.W)) 259 //reqCycleCnt := reqCycleCnt + BoolStopWatch(ptw.req(0).fire(), ptw.resp.fire || sfence.valid) 260 //XSPerfAccumulate("ptw_req_count", ptw.req.fire()) 261 //XSPerfAccumulate("ptw_req_cycle", Mux(ptw.resp.fire(), reqCycleCnt, 0.U)) 262 XSPerfAccumulate("ptw_resp_count", ptw.resp.fire()) 263 XSPerfAccumulate("ptw_resp_pf_count", ptw.resp.fire() && ptw.resp.bits.pf) 264 265 // Log 266 for(i <- 0 until Width) { 267 XSDebug(req(i).valid, p"req(${i.U}): (${req(i).valid} ${req(i).ready}) ${req(i).bits}\n") 268 XSDebug(resp(i).valid, p"resp(${i.U}): (${resp(i).valid} ${resp(i).ready}) ${resp(i).bits}\n") 269 } 270 271 XSDebug(sfence.valid, p"Sfence: ${sfence}\n") 272 XSDebug(ParallelOR(valid)|| ptw.resp.valid, p"CSR: ${csr}\n") 273 XSDebug(ParallelOR(valid) || ptw.resp.valid, p"vmEnable:${vmEnable} hit:${Binary(VecInit(hitVec).asUInt)} miss:${Binary(VecInit(missVec).asUInt)}\n") 274 for (i <- ptw.req.indices) { 275 XSDebug(ptw.req(i).fire(), p"PTW req:${ptw.req(i).bits}\n") 276 } 277 XSDebug(ptw.resp.valid, p"PTW resp:${ptw.resp.bits} (v:${ptw.resp.valid}r:${ptw.resp.ready}) \n") 278 279 println(s"${q.name}: normal page: ${q.normalNWays} ${q.normalAssociative} ${q.normalReplacer.get} super page: ${q.superNWays} ${q.superAssociative} ${q.superReplacer.get}") 280 281// // NOTE: just for simple tlb debug, comment it after tlb's debug 282 // assert(!io.ptw.resp.valid || io.ptw.resp.bits.entry.tag === io.ptw.resp.bits.entry.ppn, "Simple tlb debug requires vpn === ppn") 283 val perfinfo = IO(new Bundle(){ 284 val perfEvents = Output(new PerfEventsBundle(2)) 285 }) 286 if(!q.shouldBlock) { 287 val perfEvents = Seq( 288 ("access ", PopCount((0 until Width).map(i => vmEnable && validRegVec(i))) ), 289 ("miss ", PopCount((0 until Width).map(i => vmEnable && validRegVec(i) && missVec(i))) ), 290 ) 291 for (((perf_out,(perf_name,perf)),i) <- perfinfo.perfEvents.perf_events.zip(perfEvents).zipWithIndex) { 292 perf_out.incr_step := RegNext(perf) 293 } 294 } else { 295 val perfEvents = Seq( 296 ("access ", PopCount((0 until Width).map(i => io.requestor(i).req.fire())) ), 297 ("miss ", PopCount((0 until Width).map(i => ptw.req(i).fire())) ), 298 ) 299 for (((perf_out,(perf_name,perf)),i) <- perfinfo.perfEvents.perf_events.zip(perfEvents).zipWithIndex) { 300 perf_out.incr_step := RegNext(perf) 301 } 302 } 303} 304 305class TlbReplace(Width: Int, q: TLBParameters)(implicit p: Parameters) extends TlbModule { 306 val io = IO(new TlbReplaceIO(Width, q)) 307 308 if (q.normalAssociative == "fa") { 309 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNWays) 310 re.access(io.normalPage.access.map(_.touch_ways)) 311 io.normalPage.refillIdx := re.way 312 } else { // set-acco && plru 313 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNSets, q.normalNWays) 314 re.access(io.normalPage.access.map(_.sets), io.normalPage.access.map(_.touch_ways)) 315 io.normalPage.refillIdx := { if (q.normalNWays == 1) 0.U else re.way(io.normalPage.chosen_set) } 316 } 317 318 if (q.superAssociative == "fa") { 319 val re = ReplacementPolicy.fromString(q.superReplacer, q.superNWays) 320 re.access(io.superPage.access.map(_.touch_ways)) 321 io.superPage.refillIdx := re.way 322 } else { // set-acco && plru 323 val re = ReplacementPolicy.fromString(q.superReplacer, q.superNSets, q.superNWays) 324 re.access(io.superPage.access.map(_.sets), io.superPage.access.map(_.touch_ways)) 325 io.superPage.refillIdx := { if (q.superNWays == 1) 0.U else re.way(io.superPage.chosen_set) } 326 } 327} 328 329object TLB { 330 def apply 331 ( 332 in: Seq[BlockTlbRequestIO], 333 sfence: SfenceBundle, 334 csr: TlbCsrBundle, 335 width: Int, 336 shouldBlock: Boolean, 337 q: TLBParameters 338 )(implicit p: Parameters) = { 339 require(in.length == width) 340 341 val tlb = Module(new TLB(width, q)) 342 343 tlb.io.sfence <> sfence 344 tlb.io.csr <> csr 345 tlb.suggestName(s"tlb_${q.name}") 346 347 if (!shouldBlock) { // dtlb 348 for (i <- 0 until width) { 349 tlb.io.requestor(i) <> in(i) 350 // tlb.io.requestor(i).req.valid := in(i).req.valid 351 // tlb.io.requestor(i).req.bits := in(i).req.bits 352 // in(i).req.ready := tlb.io.requestor(i).req.ready 353 354 // in(i).resp.valid := tlb.io.requestor(i).resp.valid 355 // in(i).resp.bits := tlb.io.requestor(i).resp.bits 356 // tlb.io.requestor(i).resp.ready := in(i).resp.ready 357 } 358 } else { // itlb 359 //require(width == 1) 360 (0 until width).map{ i => 361 tlb.io.requestor(i).req.valid := in(i).req.valid 362 tlb.io.requestor(i).req.bits := in(i).req.bits 363 in(i).req.ready := !tlb.io.requestor(i).resp.bits.miss && in(i).resp.ready && tlb.io.requestor(i).req.ready 364 365 in(i).resp.valid := tlb.io.requestor(i).resp.valid && !tlb.io.requestor(i).resp.bits.miss 366 in(i).resp.bits := tlb.io.requestor(i).resp.bits 367 tlb.io.requestor(i).resp.ready := in(i).resp.ready 368 } 369 } 370 tlb.io.ptw 371 } 372} 373