1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.internal.naming.chiselName 22import chisel3.util._ 23import freechips.rocketchip.util.SRAMAnnotation 24import xiangshan._ 25import utils._ 26import xiangshan.backend.fu.{PMPChecker, PMPReqBundle} 27import xiangshan.backend.rob.RobPtr 28import xiangshan.backend.fu.util.HasCSRConst 29 30 31@chiselName 32class TLB(Width: Int, q: TLBParameters)(implicit p: Parameters) extends TlbModule with HasCSRConst { 33 val io = IO(new TlbIO(Width, q)) 34 35 require(q.superAssociative == "fa") 36 if (q.sameCycle) { 37 require(q.normalAssociative == "fa") 38 } 39 40 val req = io.requestor.map(_.req) 41 val resp = io.requestor.map(_.resp) 42 val ptw = io.ptw 43 val pmp = io.pmp 44 45 val sfence = io.sfence 46 val csr = io.csr 47 val satp = csr.satp 48 val priv = csr.priv 49 val ifecth = if (q.fetchi) true.B else false.B 50 val mode = if (q.useDmode) priv.dmode else priv.imode 51 // val vmEnable = satp.mode === 8.U // && (mode < ModeM) // FIXME: fix me when boot xv6/linux... 52 val vmEnable = if (EnbaleTlbDebug) (satp.mode === 8.U) 53 else (satp.mode === 8.U && (mode < ModeM)) 54 55 val reqAddr = req.map(_.bits.vaddr.asTypeOf((new VaBundle).cloneType)) 56 val vpn = reqAddr.map(_.vpn) 57 val cmd = req.map(_.bits.cmd) 58 val valid = req.map(_.valid) 59 60 def widthMapSeq[T <: Seq[Data]](f: Int => T) = (0 until Width).map(f) 61 62 def widthMap[T <: Data](f: Int => T) = (0 until Width).map(f) 63 64 // Normal page && Super page 65 val normalPage = TlbStorage( 66 name = "normal", 67 associative = q.normalAssociative, 68 sameCycle = q.sameCycle, 69 ports = Width, 70 nSets = q.normalNSets, 71 nWays = q.normalNWays, 72 sramSinglePort = sramSinglePort, 73 normalPage = true, 74 superPage = false 75 ) 76 val superPage = TlbStorage( 77 name = "super", 78 associative = q.superAssociative, 79 sameCycle = q.sameCycle, 80 ports = Width, 81 nSets = q.superNSets, 82 nWays = q.superNWays, 83 sramSinglePort = sramSinglePort, 84 normalPage = q.normalAsVictim, 85 superPage = true, 86 ) 87 88 89 for (i <- 0 until Width) { 90 normalPage.r_req_apply( 91 valid = io.requestor(i).req.valid, 92 vpn = vpn(i), 93 asid = csr.satp.asid, 94 i = i 95 ) 96 superPage.r_req_apply( 97 valid = io.requestor(i).req.valid, 98 vpn = vpn(i), 99 asid = csr.satp.asid, 100 i = i 101 ) 102 } 103 104 normalPage.victim.in <> superPage.victim.out 105 normalPage.victim.out <> superPage.victim.in 106 normalPage.sfence <> io.sfence 107 superPage.sfence <> io.sfence 108 normalPage.csr <> io.csr 109 superPage.csr <> io.csr 110 111 def TLBNormalRead(i: Int) = { 112 val (normal_hit, normal_ppn, normal_perm, normal_hitVec) = normalPage.r_resp_apply(i) 113 val (super_hit, super_ppn, super_perm, super_hitVec) = superPage.r_resp_apply(i) 114 assert(!(normal_hit && super_hit && vmEnable && RegNext(req(i).valid, init = false.B))) 115 116 val hit = normal_hit || super_hit 117 val ppn = Mux(normal_hit, normal_ppn, super_ppn) 118 val perm = Mux(normal_hit, normal_perm, super_perm) 119 120 val pf = perm.pf && hit 121 val af = perm.af && hit 122 val cmdReg = if (!q.sameCycle) RegNext(cmd(i)) else cmd(i) 123 val validReg = if (!q.sameCycle) RegNext(valid(i)) else valid(i) 124 val offReg = if (!q.sameCycle) RegNext(reqAddr(i).off) else reqAddr(i).off 125 val sizeReg = if (!q.sameCycle) RegNext(req(i).bits.size) else req(i).bits.size 126 127 /** *************** next cycle when two cycle is false******************* */ 128 val miss = !hit && vmEnable 129 hit.suggestName(s"hit_${i}") 130 miss.suggestName(s"miss_${i}") 131 132 XSDebug(validReg, p"(${i.U}) hit:${hit} miss:${miss} ppn:${Hexadecimal(ppn)} perm:${perm}\n") 133 134 val paddr = Cat(ppn, offReg) 135 val vaddr = SignExt(req(i).bits.vaddr, PAddrBits) 136 137 req(i).ready := resp(i).ready 138 resp(i).valid := validReg 139 resp(i).bits.paddr := Mux(vmEnable, paddr, if (!q.sameCycle) RegNext(vaddr) else vaddr) 140 resp(i).bits.miss := miss 141 resp(i).bits.ptwBack := io.ptw.resp.fire() 142 143 pmp(i).valid := resp(i).valid 144 pmp(i).bits.addr := resp(i).bits.paddr 145 pmp(i).bits.size := sizeReg 146 pmp(i).bits.cmd := cmdReg 147 148 val update = hit && (!perm.a || !perm.d && TlbCmd.isWrite(cmdReg)) // update A/D through exception 149 val modeCheck = !(mode === ModeU && !perm.u || mode === ModeS && perm.u && (!priv.sum || ifecth)) 150 val ldPf = !(modeCheck && (perm.r || priv.mxr && perm.x)) && (TlbCmd.isRead(cmdReg) && true.B /* TODO !isAMO*/) 151 val stPf = !(modeCheck && perm.w) && (TlbCmd.isWrite(cmdReg) || false.B /*TODO isAMO. */) 152 val instrPf = !(modeCheck && perm.x) && TlbCmd.isExec(cmdReg) 153 resp(i).bits.excp.pf.ld := (ldPf || update || pf) && vmEnable && hit && !af 154 resp(i).bits.excp.pf.st := (stPf || update || pf) && vmEnable && hit && !af 155 resp(i).bits.excp.pf.instr := (instrPf || update || pf) && vmEnable && hit && !af 156 // NOTE: pf need && with !af, page fault has higher priority than access fault 157 // but ptw may also have access fault, then af happens, the translation is wrong. 158 // In this case, pf has lower priority than af 159 160 // if vmenable, use pre-calcuated pma check result 161 resp(i).bits.mmio := Mux(TlbCmd.isExec(cmdReg), !perm.pi, !perm.pd) && vmEnable && hit 162 resp(i).bits.excp.af.ld := (af || Mux(TlbCmd.isAtom(cmdReg), !perm.pa, !perm.pr) && TlbCmd.isRead(cmdReg)) && vmEnable && hit 163 resp(i).bits.excp.af.st := (af || Mux(TlbCmd.isAtom(cmdReg), !perm.pa, !perm.pw) && TlbCmd.isWrite(cmdReg)) && vmEnable && hit 164 resp(i).bits.excp.af.instr := (af || Mux(TlbCmd.isAtom(cmdReg), false.B, !perm.pe)) && vmEnable && hit 165 166 // if !vmenable, check pma 167 val (pmaMode, accessWidth) = AddressSpace.memmapAddrMatch(resp(i).bits.paddr) 168 when(!vmEnable) { 169 resp(i).bits.mmio := Mux(TlbCmd.isExec(cmdReg), !PMAMode.icache(pmaMode), !PMAMode.dcache(pmaMode)) 170 resp(i).bits.excp.af.ld := Mux(TlbCmd.isAtom(cmdReg), !PMAMode.atomic(pmaMode), !PMAMode.read(pmaMode)) && TlbCmd.isRead(cmdReg) 171 resp(i).bits.excp.af.st := Mux(TlbCmd.isAtom(cmdReg), !PMAMode.atomic(pmaMode), !PMAMode.write(pmaMode)) && TlbCmd.isWrite(cmdReg) 172 resp(i).bits.excp.af.instr := Mux(TlbCmd.isAtom(cmdReg), false.B, !PMAMode.execute(pmaMode)) 173 } 174 175 (hit, miss, normal_hitVec, super_hitVec, validReg) 176 } 177 178 val readResult = (0 until Width).map(TLBNormalRead(_)) 179 val hitVec = readResult.map(_._1) 180 val missVec = readResult.map(_._2) 181 val normalhitVecVec = readResult.map(_._3) 182 val superhitVecVec = readResult.map(_._4) 183 val validRegVec = readResult.map(_._5) 184 185 // replacement 186 def get_access(one_hot: UInt, valid: Bool): Valid[UInt] = { 187 val res = Wire(Valid(UInt(log2Up(one_hot.getWidth).W))) 188 res.valid := Cat(one_hot).orR && valid 189 res.bits := OHToUInt(one_hot) 190 res 191 } 192 193 val normal_refill_idx = if (q.outReplace) { 194 io.replace.normalPage.access.sets := vpn.map(get_idx(_, q.normalNSets)) 195 io.replace.normalPage.access.touch_ways := normalhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, 196 validRegVec(i))} 197 io.replace.normalPage.chosen_set := get_idx(io.ptw.resp.bits.entry.tag, q.normalNSets) 198 io.replace.normalPage.refillIdx 199 } else if (q.normalAssociative == "fa") { 200 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNWays) 201 re.access(normalhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, validRegVec(i))}) 202 re.way 203 } else { // set-acco && plru 204 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNSets, q.normalNWays) 205 re.access(vpn.map(get_idx(_, q.normalNSets)), normalhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, 206 validRegVec(i))}) 207 re.way(get_idx(io.ptw.resp.bits.entry.tag, q.normalNSets)) 208 } 209 210 val super_refill_idx = if (q.outReplace) { 211 io.replace.superPage.access.sets := vpn.map(get_idx(_, q.normalNSets)) 212 io.replace.superPage.access.touch_ways := superhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, 213 validRegVec(i))} 214 io.replace.superPage.chosen_set := DontCare 215 io.replace.superPage.refillIdx 216 } else { 217 val re = ReplacementPolicy.fromString(q.superReplacer, q.superNWays) 218 re.access(superhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, validRegVec(i))}) 219 re.way 220 } 221 222 val refill = ptw.resp.fire() && !sfence.valid && !satp.changed 223 normalPage.w_apply( 224 valid = { if (q.normalAsVictim) false.B 225 else refill && ptw.resp.bits.entry.level.get === 2.U }, 226 wayIdx = normal_refill_idx, 227 data = ptw.resp.bits 228 ) 229 superPage.w_apply( 230 valid = { if (q.normalAsVictim) refill 231 else refill && ptw.resp.bits.entry.level.get =/= 2.U }, 232 wayIdx = super_refill_idx, 233 data = ptw.resp.bits 234 ) 235 236 for (i <- 0 until Width) { 237 io.ptw.req(i).valid := validRegVec(i) && missVec(i) && !RegNext(refill) 238 io.ptw.req(i).bits.vpn := RegNext(reqAddr(i).vpn) 239 } 240 io.ptw.resp.ready := true.B 241 242 if (!q.shouldBlock) { 243 for (i <- 0 until Width) { 244 XSPerfAccumulate("first_access" + Integer.toString(i, 10), validRegVec(i) && vmEnable && RegNext(req(i).bits.debug.isFirstIssue)) 245 XSPerfAccumulate("access" + Integer.toString(i, 10), validRegVec(i) && vmEnable) 246 } 247 for (i <- 0 until Width) { 248 XSPerfAccumulate("first_miss" + Integer.toString(i, 10), validRegVec(i) && vmEnable && missVec(i) && RegNext(req(i).bits.debug.isFirstIssue)) 249 XSPerfAccumulate("miss" + Integer.toString(i, 10), validRegVec(i) && vmEnable && missVec(i)) 250 } 251 } else { 252 // NOTE: ITLB is blocked, so every resp will be valid only when hit 253 // every req will be ready only when hit 254 for (i <- 0 until Width) { 255 XSPerfAccumulate(s"access${i}", io.requestor(i).req.fire() && vmEnable) 256 XSPerfAccumulate(s"miss${i}", ptw.req(i).fire()) 257 } 258 259 } 260 //val reqCycleCnt = Reg(UInt(16.W)) 261 //reqCycleCnt := reqCycleCnt + BoolStopWatch(ptw.req(0).fire(), ptw.resp.fire || sfence.valid) 262 //XSPerfAccumulate("ptw_req_count", ptw.req.fire()) 263 //XSPerfAccumulate("ptw_req_cycle", Mux(ptw.resp.fire(), reqCycleCnt, 0.U)) 264 XSPerfAccumulate("ptw_resp_count", ptw.resp.fire()) 265 XSPerfAccumulate("ptw_resp_pf_count", ptw.resp.fire() && ptw.resp.bits.pf) 266 267 // Log 268 for(i <- 0 until Width) { 269 XSDebug(req(i).valid, p"req(${i.U}): (${req(i).valid} ${req(i).ready}) ${req(i).bits}\n") 270 XSDebug(resp(i).valid, p"resp(${i.U}): (${resp(i).valid} ${resp(i).ready}) ${resp(i).bits}\n") 271 } 272 273 XSDebug(sfence.valid, p"Sfence: ${sfence}\n") 274 XSDebug(ParallelOR(valid)|| ptw.resp.valid, p"CSR: ${csr}\n") 275 XSDebug(ParallelOR(valid) || ptw.resp.valid, p"vmEnable:${vmEnable} hit:${Binary(VecInit(hitVec).asUInt)} miss:${Binary(VecInit(missVec).asUInt)}\n") 276 for (i <- ptw.req.indices) { 277 XSDebug(ptw.req(i).fire(), p"PTW req:${ptw.req(i).bits}\n") 278 } 279 XSDebug(ptw.resp.valid, p"PTW resp:${ptw.resp.bits} (v:${ptw.resp.valid}r:${ptw.resp.ready}) \n") 280 281 println(s"${q.name}: normal page: ${q.normalNWays} ${q.normalAssociative} ${q.normalReplacer.get} super page: ${q.superNWays} ${q.superAssociative} ${q.superReplacer.get}") 282 283// // NOTE: just for simple tlb debug, comment it after tlb's debug 284 // assert(!io.ptw.resp.valid || io.ptw.resp.bits.entry.tag === io.ptw.resp.bits.entry.ppn, "Simple tlb debug requires vpn === ppn") 285} 286 287class TlbReplace(Width: Int, q: TLBParameters)(implicit p: Parameters) extends TlbModule { 288 val io = IO(new TlbReplaceIO(Width, q)) 289 290 if (q.normalAssociative == "fa") { 291 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNWays) 292 re.access(io.normalPage.access.touch_ways) 293 io.normalPage.refillIdx := re.way 294 } else { // set-acco && plru 295 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNSets, q.normalNWays) 296 re.access(io.normalPage.access.sets, io.normalPage.access.touch_ways) 297 io.normalPage.refillIdx := { if (q.normalNWays == 1) 0.U else re.way(io.normalPage.chosen_set) } 298 } 299 300 if (q.superAssociative == "fa") { 301 val re = ReplacementPolicy.fromString(q.superReplacer, q.superNWays) 302 re.access(io.superPage.access.touch_ways) 303 io.superPage.refillIdx := re.way 304 } else { // set-acco && plru 305 val re = ReplacementPolicy.fromString(q.superReplacer, q.superNSets, q.superNWays) 306 re.access(io.superPage.access.sets, io.superPage.access.touch_ways) 307 io.superPage.refillIdx := { if (q.superNWays == 1) 0.U else re.way(io.superPage.chosen_set) } 308 } 309} 310 311object TLB { 312 def apply 313 ( 314 in: Seq[BlockTlbRequestIO], 315 sfence: SfenceBundle, 316 csr: TlbCsrBundle, 317 width: Int, 318 shouldBlock: Boolean, 319 q: TLBParameters 320 )(implicit p: Parameters) = { 321 require(in.length == width) 322 323 val tlb = Module(new TLB(width, q)) 324 325 tlb.io.sfence <> sfence 326 tlb.io.csr <> csr 327 tlb.suggestName(s"tlb_${q.name}") 328 329 if (!shouldBlock) { // dtlb 330 for (i <- 0 until width) { 331 tlb.io.requestor(i) <> in(i) 332 // tlb.io.requestor(i).req.valid := in(i).req.valid 333 // tlb.io.requestor(i).req.bits := in(i).req.bits 334 // in(i).req.ready := tlb.io.requestor(i).req.ready 335 336 // in(i).resp.valid := tlb.io.requestor(i).resp.valid 337 // in(i).resp.bits := tlb.io.requestor(i).resp.bits 338 // tlb.io.requestor(i).resp.ready := in(i).resp.ready 339 } 340 } else { // itlb 341 //require(width == 1) 342 (0 until width).map{ i => 343 tlb.io.requestor(i).req.valid := in(i).req.valid 344 tlb.io.requestor(i).req.bits := in(i).req.bits 345 in(i).req.ready := !tlb.io.requestor(i).resp.bits.miss && in(i).resp.ready && tlb.io.requestor(i).req.ready 346 347 in(i).resp.valid := tlb.io.requestor(i).resp.valid && !tlb.io.requestor(i).resp.bits.miss 348 in(i).resp.bits := tlb.io.requestor(i).resp.bits 349 tlb.io.requestor(i).resp.ready := in(i).resp.ready 350 } 351 } 352 353 tlb.io.ptw 354 } 355} 356