1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache.mmu 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.internal.naming.chiselName 22import chisel3.util._ 23import freechips.rocketchip.util.SRAMAnnotation 24import xiangshan._ 25import utils._ 26import xiangshan.backend.fu.{PMPChecker, PMPReqBundle} 27import xiangshan.backend.rob.RobPtr 28import xiangshan.backend.fu.util.HasCSRConst 29 30 31@chiselName 32class TLB(Width: Int, q: TLBParameters)(implicit p: Parameters) extends TlbModule with HasCSRConst { 33 val io = IO(new TlbIO(Width, q)) 34 35 require(q.superAssociative == "fa") 36 if (q.sameCycle) { 37 require(q.normalAssociative == "fa") 38 } 39 40 val req = io.requestor.map(_.req) 41 val resp = io.requestor.map(_.resp) 42 val ptw = io.ptw 43 val pmp = io.pmp 44 45 val sfence = io.sfence 46 val csr = io.csr 47 val satp = csr.satp 48 val priv = csr.priv 49 val ifecth = if (q.fetchi) true.B else false.B 50 val mode = if (q.useDmode) priv.dmode else priv.imode 51 // val vmEnable = satp.mode === 8.U // && (mode < ModeM) // FIXME: fix me when boot xv6/linux... 52 val vmEnable = if (EnbaleTlbDebug) (satp.mode === 8.U) 53 else (satp.mode === 8.U && (mode < ModeM)) 54 55 val reqAddr = req.map(_.bits.vaddr.asTypeOf((new VaBundle).cloneType)) 56 val vpn = reqAddr.map(_.vpn) 57 val cmd = req.map(_.bits.cmd) 58 val valid = req.map(_.valid) 59 60 def widthMapSeq[T <: Seq[Data]](f: Int => T) = (0 until Width).map(f) 61 62 def widthMap[T <: Data](f: Int => T) = (0 until Width).map(f) 63 64 // Normal page && Super page 65 val normalPage = TlbStorage( 66 name = "normal", 67 associative = q.normalAssociative, 68 sameCycle = q.sameCycle, 69 ports = Width, 70 nSets = q.normalNSets, 71 nWays = q.normalNWays, 72 sramSinglePort = sramSinglePort, 73 normalPage = true, 74 superPage = false 75 ) 76 val superPage = TlbStorage( 77 name = "super", 78 associative = q.superAssociative, 79 sameCycle = q.sameCycle, 80 ports = Width, 81 nSets = q.superNSets, 82 nWays = q.superNWays, 83 sramSinglePort = sramSinglePort, 84 normalPage = q.normalAsVictim, 85 superPage = true, 86 ) 87 88 89 for (i <- 0 until Width) { 90 normalPage.r_req_apply( 91 valid = io.requestor(i).req.valid, 92 vpn = vpn(i), 93 i = i 94 ) 95 superPage.r_req_apply( 96 valid = io.requestor(i).req.valid, 97 vpn = vpn(i), 98 i = i 99 ) 100 } 101 102 103 normalPage.victim.in <> superPage.victim.out 104 normalPage.victim.out <> superPage.victim.in 105 normalPage.sfence <> io.sfence 106 superPage.sfence <> io.sfence 107 108 def TLBNormalRead(i: Int) = { 109 val (normal_hit, normal_ppn, normal_perm, normal_hitVec) = normalPage.r_resp_apply(i) 110 val (super_hit, super_ppn, super_perm, super_hitVec) = superPage.r_resp_apply(i) 111 assert(!(normal_hit && super_hit && vmEnable && RegNext(req(i).valid, init = false.B))) 112 113 val hit = normal_hit || super_hit 114 val ppn = Mux(normal_hit, normal_ppn, super_ppn) 115 val perm = Mux(normal_hit, normal_perm, super_perm) 116 117 val pf = perm.pf && hit 118 val af = perm.af && hit 119 val cmdReg = if (!q.sameCycle) RegNext(cmd(i)) else cmd(i) 120 val validReg = if (!q.sameCycle) RegNext(valid(i)) else valid(i) 121 val offReg = if (!q.sameCycle) RegNext(reqAddr(i).off) else reqAddr(i).off 122 val sizeReg = if (!q.sameCycle) RegNext(req(i).bits.size) else req(i).bits.size 123 124 /** *************** next cycle when two cycle is false******************* */ 125 val miss = !hit && vmEnable 126 hit.suggestName(s"hit_${i}") 127 miss.suggestName(s"miss_${i}") 128 129 XSDebug(validReg, p"(${i.U}) hit:${hit} miss:${miss} ppn:${Hexadecimal(ppn)} perm:${perm}\n") 130 131 val paddr = Cat(ppn, offReg) 132 val vaddr = SignExt(req(i).bits.vaddr, PAddrBits) 133 134 req(i).ready := resp(i).ready 135 resp(i).valid := validReg 136 resp(i).bits.paddr := Mux(vmEnable, paddr, if (!q.sameCycle) RegNext(vaddr) else vaddr) 137 resp(i).bits.miss := miss 138 resp(i).bits.ptwBack := io.ptw.resp.fire() 139 140 pmp(i).valid := resp(i).valid 141 pmp(i).bits.addr := resp(i).bits.paddr 142 pmp(i).bits.size := sizeReg 143 pmp(i).bits.cmd := cmdReg 144 145 val update = hit && (!perm.a || !perm.d && TlbCmd.isWrite(cmdReg)) // update A/D through exception 146 val modeCheck = !(mode === ModeU && !perm.u || mode === ModeS && perm.u && (!priv.sum || ifecth)) 147 val ldPf = !(modeCheck && (perm.r || priv.mxr && perm.x)) && (TlbCmd.isRead(cmdReg) && true.B /* TODO !isAMO*/) 148 val stPf = !(modeCheck && perm.w) && (TlbCmd.isWrite(cmdReg) || false.B /*TODO isAMO. */) 149 val instrPf = !(modeCheck && perm.x) && TlbCmd.isExec(cmdReg) 150 resp(i).bits.excp.pf.ld := (ldPf || update || pf) && vmEnable && hit && !af 151 resp(i).bits.excp.pf.st := (stPf || update || pf) && vmEnable && hit && !af 152 resp(i).bits.excp.pf.instr := (instrPf || update || pf) && vmEnable && hit && !af 153 // NOTE: pf need && with !af, page fault has higher priority than access fault 154 // but ptw may also have access fault, then af happens, the translation is wrong. 155 // In this case, pf has lower priority than af 156 157 // if vmenable, use pre-calcuated pma check result 158 resp(i).bits.mmio := Mux(TlbCmd.isExec(cmdReg), !perm.pi, !perm.pd) && vmEnable && hit 159 resp(i).bits.excp.af.ld := (af || Mux(TlbCmd.isAtom(cmdReg), !perm.pa, !perm.pr) && TlbCmd.isRead(cmdReg)) && vmEnable && hit 160 resp(i).bits.excp.af.st := (af || Mux(TlbCmd.isAtom(cmdReg), !perm.pa, !perm.pw) && TlbCmd.isWrite(cmdReg)) && vmEnable && hit 161 resp(i).bits.excp.af.instr := (af || Mux(TlbCmd.isAtom(cmdReg), false.B, !perm.pe)) && vmEnable && hit 162 163 // if !vmenable, check pma 164 val (pmaMode, accessWidth) = AddressSpace.memmapAddrMatch(resp(i).bits.paddr) 165 when(!vmEnable) { 166 resp(i).bits.mmio := Mux(TlbCmd.isExec(cmdReg), !PMAMode.icache(pmaMode), !PMAMode.dcache(pmaMode)) 167 resp(i).bits.excp.af.ld := Mux(TlbCmd.isAtom(cmdReg), !PMAMode.atomic(pmaMode), !PMAMode.read(pmaMode)) && TlbCmd.isRead(cmdReg) 168 resp(i).bits.excp.af.st := Mux(TlbCmd.isAtom(cmdReg), !PMAMode.atomic(pmaMode), !PMAMode.write(pmaMode)) && TlbCmd.isWrite(cmdReg) 169 resp(i).bits.excp.af.instr := Mux(TlbCmd.isAtom(cmdReg), false.B, !PMAMode.execute(pmaMode)) 170 } 171 172 (hit, miss, normal_hitVec, super_hitVec, validReg) 173 } 174 175 val readResult = (0 until Width).map(TLBNormalRead(_)) 176 val hitVec = readResult.map(_._1) 177 val missVec = readResult.map(_._2) 178 val normalhitVecVec = readResult.map(_._3) 179 val superhitVecVec = readResult.map(_._4) 180 val validRegVec = readResult.map(_._5) 181 182 // replacement 183 def get_access(one_hot: UInt, valid: Bool): Valid[UInt] = { 184 val res = Wire(Valid(UInt(log2Up(one_hot.getWidth).W))) 185 res.valid := Cat(one_hot).orR && valid 186 res.bits := OHToUInt(one_hot) 187 res 188 } 189 190 val normal_refill_idx = if (q.outReplace) { 191 io.replace.normalPage.access.sets := vpn.map(get_idx(_, q.normalNSets)) 192 io.replace.normalPage.access.touch_ways := normalhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, 193 validRegVec(i))} 194 io.replace.normalPage.chosen_set := get_idx(io.ptw.resp.bits.entry.tag, q.normalNSets) 195 io.replace.normalPage.refillIdx 196 } else if (q.normalAssociative == "fa") { 197 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNWays) 198 re.access(normalhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, validRegVec(i))}) 199 re.way 200 } else { // set-acco && plru 201 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNSets, q.normalNWays) 202 re.access(vpn.map(get_idx(_, q.normalNSets)), normalhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, 203 validRegVec(i))}) 204 re.way(get_idx(io.ptw.resp.bits.entry.tag, q.normalNSets)) 205 } 206 207 val super_refill_idx = if (q.outReplace) { 208 io.replace.superPage.access.sets := vpn.map(get_idx(_, q.normalNSets)) 209 io.replace.superPage.access.touch_ways := superhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, 210 validRegVec(i))} 211 io.replace.superPage.chosen_set := DontCare 212 io.replace.superPage.refillIdx 213 } else { 214 val re = ReplacementPolicy.fromString(q.superReplacer, q.superNWays) 215 re.access(superhitVecVec.zipWithIndex.map{ case (hv, i) => get_access(hv, validRegVec(i))}) 216 re.way 217 } 218 219 val refill = ptw.resp.fire() && !sfence.valid 220 normalPage.w_apply( 221 valid = { if (q.normalAsVictim) false.B 222 else refill && ptw.resp.bits.entry.level.get === 2.U }, 223 wayIdx = normal_refill_idx, 224 data = ptw.resp.bits 225 ) 226 superPage.w_apply( 227 valid = { if (q.normalAsVictim) refill 228 else refill && ptw.resp.bits.entry.level.get =/= 2.U }, 229 wayIdx = super_refill_idx, 230 data = ptw.resp.bits 231 ) 232 233 for (i <- 0 until Width) { 234 io.ptw.req(i).valid := validRegVec(i) && missVec(i) && !RegNext(refill) 235 io.ptw.req(i).bits.vpn := RegNext(reqAddr(i).vpn) 236 } 237 io.ptw.resp.ready := true.B 238 239 if (!q.shouldBlock) { 240 for (i <- 0 until Width) { 241 XSPerfAccumulate("first_access" + Integer.toString(i, 10), validRegVec(i) && vmEnable && RegNext(req(i).bits.debug.isFirstIssue)) 242 XSPerfAccumulate("access" + Integer.toString(i, 10), validRegVec(i) && vmEnable) 243 } 244 for (i <- 0 until Width) { 245 XSPerfAccumulate("first_miss" + Integer.toString(i, 10), validRegVec(i) && vmEnable && missVec(i) && RegNext(req(i).bits.debug.isFirstIssue)) 246 XSPerfAccumulate("miss" + Integer.toString(i, 10), validRegVec(i) && vmEnable && missVec(i)) 247 } 248 } else { 249 // NOTE: ITLB is blocked, so every resp will be valid only when hit 250 // every req will be ready only when hit 251 for (i <- 0 until Width) { 252 XSPerfAccumulate(s"access${i}", io.requestor(i).req.fire() && vmEnable) 253 XSPerfAccumulate(s"miss${i}", ptw.req(i).fire()) 254 } 255 256 } 257 //val reqCycleCnt = Reg(UInt(16.W)) 258 //reqCycleCnt := reqCycleCnt + BoolStopWatch(ptw.req(0).fire(), ptw.resp.fire || sfence.valid) 259 //XSPerfAccumulate("ptw_req_count", ptw.req.fire()) 260 //XSPerfAccumulate("ptw_req_cycle", Mux(ptw.resp.fire(), reqCycleCnt, 0.U)) 261 XSPerfAccumulate("ptw_resp_count", ptw.resp.fire()) 262 XSPerfAccumulate("ptw_resp_pf_count", ptw.resp.fire() && ptw.resp.bits.pf) 263 264 // Log 265 for(i <- 0 until Width) { 266 XSDebug(req(i).valid, p"req(${i.U}): (${req(i).valid} ${req(i).ready}) ${req(i).bits}\n") 267 XSDebug(resp(i).valid, p"resp(${i.U}): (${resp(i).valid} ${resp(i).ready}) ${resp(i).bits}\n") 268 } 269 270 XSDebug(sfence.valid, p"Sfence: ${sfence}\n") 271 XSDebug(ParallelOR(valid)|| ptw.resp.valid, p"CSR: ${csr}\n") 272 XSDebug(ParallelOR(valid) || ptw.resp.valid, p"vmEnable:${vmEnable} hit:${Binary(VecInit(hitVec).asUInt)} miss:${Binary(VecInit(missVec).asUInt)}\n") 273 for (i <- ptw.req.indices) { 274 XSDebug(ptw.req(i).fire(), p"PTW req:${ptw.req(i).bits}\n") 275 } 276 XSDebug(ptw.resp.valid, p"PTW resp:${ptw.resp.bits} (v:${ptw.resp.valid}r:${ptw.resp.ready}) \n") 277 278 println(s"${q.name}: normal page: ${q.normalNWays} ${q.normalAssociative} ${q.normalReplacer.get} super page: ${q.superNWays} ${q.superAssociative} ${q.superReplacer.get}") 279 280// // NOTE: just for simple tlb debug, comment it after tlb's debug 281 // assert(!io.ptw.resp.valid || io.ptw.resp.bits.entry.tag === io.ptw.resp.bits.entry.ppn, "Simple tlb debug requires vpn === ppn") 282} 283 284class TlbReplace(Width: Int, q: TLBParameters)(implicit p: Parameters) extends TlbModule { 285 val io = IO(new TlbReplaceIO(Width, q)) 286 287 if (q.normalAssociative == "fa") { 288 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNWays) 289 re.access(io.normalPage.access.touch_ways) 290 io.normalPage.refillIdx := re.way 291 } else { // set-acco && plru 292 val re = ReplacementPolicy.fromString(q.normalReplacer, q.normalNSets, q.normalNWays) 293 re.access(io.normalPage.access.sets, io.normalPage.access.touch_ways) 294 io.normalPage.refillIdx := { if (q.normalNWays == 1) 0.U else re.way(io.normalPage.chosen_set) } 295 } 296 297 if (q.superAssociative == "fa") { 298 val re = ReplacementPolicy.fromString(q.superReplacer, q.superNWays) 299 re.access(io.superPage.access.touch_ways) 300 io.superPage.refillIdx := re.way 301 } else { // set-acco && plru 302 val re = ReplacementPolicy.fromString(q.superReplacer, q.superNSets, q.superNWays) 303 re.access(io.superPage.access.sets, io.superPage.access.touch_ways) 304 io.superPage.refillIdx := { if (q.superNWays == 1) 0.U else re.way(io.superPage.chosen_set) } 305 } 306} 307 308object TLB { 309 def apply 310 ( 311 in: Seq[BlockTlbRequestIO], 312 sfence: SfenceBundle, 313 csr: TlbCsrBundle, 314 width: Int, 315 shouldBlock: Boolean, 316 q: TLBParameters 317 )(implicit p: Parameters) = { 318 require(in.length == width) 319 320 val tlb = Module(new TLB(width, q)) 321 322 tlb.io.sfence <> sfence 323 tlb.io.csr <> csr 324 tlb.suggestName(s"tlb_${q.name}") 325 326 if (!shouldBlock) { // dtlb 327 for (i <- 0 until width) { 328 tlb.io.requestor(i) <> in(i) 329 // tlb.io.requestor(i).req.valid := in(i).req.valid 330 // tlb.io.requestor(i).req.bits := in(i).req.bits 331 // in(i).req.ready := tlb.io.requestor(i).req.ready 332 333 // in(i).resp.valid := tlb.io.requestor(i).resp.valid 334 // in(i).resp.bits := tlb.io.requestor(i).resp.bits 335 // tlb.io.requestor(i).resp.ready := in(i).resp.ready 336 } 337 } else { // itlb 338 //require(width == 1) 339 (0 until width).map{ i => 340 tlb.io.requestor(i).req.valid := in(i).req.valid 341 tlb.io.requestor(i).req.bits := in(i).req.bits 342 in(i).req.ready := !tlb.io.requestor(i).resp.bits.miss && in(i).resp.ready && tlb.io.requestor(i).req.ready 343 344 in(i).resp.valid := tlb.io.requestor(i).resp.valid && !tlb.io.requestor(i).resp.bits.miss 345 in(i).resp.bits := tlb.io.requestor(i).resp.bits 346 tlb.io.requestor(i).resp.ready := in(i).resp.ready 347 } 348 } 349 350 tlb.io.ptw 351 } 352} 353