xref: /XiangShan/src/main/scala/xiangshan/frontend/icache/ICache.scala (revision c41f725a91c55e75c95c55b4bb0d2649f43e4c83)
1/***************************************************************************************
2* Copyright (c) 2024 Beijing Institute of Open Source Chip (BOSC)
3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences
4* Copyright (c) 2020-2021 Peng Cheng Laboratory
5*
6* XiangShan is licensed under Mulan PSL v2.
7* You can use this software according to the terms and conditions of the Mulan PSL v2.
8* You may obtain a copy of Mulan PSL v2 at:
9*          http://license.coscl.org.cn/MulanPSL2
10*
11* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14*
15* See the Mulan PSL v2 for more details.
16*
17*
18* Acknowledgement
19*
20* This implementation is inspired by several key papers:
21* [1] Glenn Reinman, Brad Calder, and Todd Austin. "[Fetch directed instruction prefetching.]
22* (https://doi.org/10.1109/MICRO.1999.809439)" 32nd Annual ACM/IEEE International Symposium on Microarchitecture
23* (MICRO). 1999.
24***************************************************************************************/
25
26package xiangshan.frontend.icache
27
28import chisel3._
29import chisel3.util._
30import freechips.rocketchip.diplomacy.AddressSet
31import freechips.rocketchip.diplomacy.IdRange
32import freechips.rocketchip.diplomacy.LazyModule
33import freechips.rocketchip.diplomacy.LazyModuleImp
34import freechips.rocketchip.tilelink._
35import freechips.rocketchip.util.BundleFieldBase
36import huancun.AliasField
37import huancun.PrefetchField
38import org.chipsalliance.cde.config.Parameters
39import utility._
40import utils._
41import xiangshan._
42import xiangshan.cache._
43import xiangshan.cache.mmu.TlbRequestIO
44import xiangshan.frontend._
45
46case class ICacheParameters(
47    nSets:               Int = 256,
48    nWays:               Int = 4,
49    rowBits:             Int = 64,
50    nTLBEntries:         Int = 32,
51    tagECC:              Option[String] = None,
52    dataECC:             Option[String] = None,
53    replacer:            Option[String] = Some("random"),
54    PortNumber:          Int = 2,
55    nFetchMshr:          Int = 4,
56    nPrefetchMshr:       Int = 10,
57    nWayLookupSize:      Int = 32,
58    DataCodeUnit:        Int = 64,
59    ICacheDataBanks:     Int = 8,
60    ICacheDataSRAMWidth: Int = 66,
61    // TODO: hard code, need delete
62    partWayNum:          Int = 4,
63    nMMIOs:              Int = 1,
64    blockBytes:          Int = 64,
65    cacheCtrlAddressOpt: Option[AddressSet] = None
66) extends L1CacheParameters {
67
68  val setBytes:     Int         = nSets * blockBytes
69  val aliasBitsOpt: Option[Int] = Option.when(setBytes > pageSize)(log2Ceil(setBytes / pageSize))
70  val reqFields: Seq[BundleFieldBase] = Seq(
71    PrefetchField(),
72    ReqSourceField()
73  ) ++ aliasBitsOpt.map(AliasField)
74  val echoFields: Seq[BundleFieldBase] = Nil
75  def tagCode:    Code                 = Code.fromString(tagECC)
76  def dataCode:   Code                 = Code.fromString(dataECC)
77  def replacement = ReplacementPolicy.fromString(replacer, nWays, nSets)
78}
79
80trait HasICacheParameters extends HasL1CacheParameters with HasInstrMMIOConst with HasIFUConst {
81  val cacheParams: ICacheParameters = icacheParameters
82
83  def ctrlUnitParamsOpt: Option[L1ICacheCtrlParams] = OptionWrapper(
84    cacheParams.cacheCtrlAddressOpt.nonEmpty,
85    L1ICacheCtrlParams(
86      address = cacheParams.cacheCtrlAddressOpt.get,
87      regWidth = XLEN
88    )
89  )
90
91  def ICacheSets:          Int = cacheParams.nSets
92  def ICacheWays:          Int = cacheParams.nWays
93  def PortNumber:          Int = cacheParams.PortNumber
94  def nFetchMshr:          Int = cacheParams.nFetchMshr
95  def nPrefetchMshr:       Int = cacheParams.nPrefetchMshr
96  def nWayLookupSize:      Int = cacheParams.nWayLookupSize
97  def DataCodeUnit:        Int = cacheParams.DataCodeUnit
98  def ICacheDataBanks:     Int = cacheParams.ICacheDataBanks
99  def ICacheDataSRAMWidth: Int = cacheParams.ICacheDataSRAMWidth
100  def partWayNum:          Int = cacheParams.partWayNum
101
102  def ICacheMetaBits:      Int = tagBits // FIXME: unportable: maybe use somemethod to get width
103  def ICacheMetaCodeBits:  Int = 1       // FIXME: unportable: maybe use cacheParams.tagCode.somemethod to get width
104  def ICacheMetaEntryBits: Int = ICacheMetaBits + ICacheMetaCodeBits
105
106  def ICacheDataBits: Int = blockBits / ICacheDataBanks
107  def ICacheDataCodeSegs: Int =
108    math.ceil(ICacheDataBits / DataCodeUnit).toInt // split data to segments for ECC checking
109  def ICacheDataCodeBits: Int =
110    ICacheDataCodeSegs * 1 // FIXME: unportable: maybe use cacheParams.dataCode.somemethod to get width
111  def ICacheDataEntryBits: Int = ICacheDataBits + ICacheDataCodeBits
112  def ICacheBankVisitNum:  Int = 32 * 8 / ICacheDataBits + 1
113  def highestIdxBit:       Int = log2Ceil(nSets) - 1
114
115  require((ICacheDataBanks >= 2) && isPow2(ICacheDataBanks))
116  require(ICacheDataSRAMWidth >= ICacheDataEntryBits)
117  require(isPow2(ICacheSets), s"nSets($ICacheSets) must be pow2")
118  require(isPow2(ICacheWays), s"nWays($ICacheWays) must be pow2")
119
120  def generatePipeControl(lastFire: Bool, thisFire: Bool, thisFlush: Bool, lastFlush: Bool): Bool = {
121    val valid = RegInit(false.B)
122    when(thisFlush)(valid := false.B)
123      .elsewhen(lastFire && !lastFlush)(valid := true.B)
124      .elsewhen(thisFire)(valid := false.B)
125    valid
126  }
127
128  def ResultHoldBypass[T <: Data](data: T, valid: Bool): T =
129    Mux(valid, data, RegEnable(data, valid))
130
131  def ResultHoldBypass[T <: Data](data: T, init: T, valid: Bool): T =
132    Mux(valid, data, RegEnable(data, init, valid))
133
134  def holdReleaseLatch(valid: Bool, release: Bool, flush: Bool): Bool = {
135    val bit = RegInit(false.B)
136    when(flush)(bit := false.B)
137      .elsewhen(valid && !release)(bit := true.B)
138      .elsewhen(release)(bit := false.B)
139    bit || valid
140  }
141
142  def blockCounter(block: Bool, flush: Bool, threshold: Int): Bool = {
143    val counter = RegInit(0.U(log2Up(threshold + 1).W))
144    when(block)(counter := counter + 1.U)
145    when(flush)(counter := 0.U)
146    counter > threshold.U
147  }
148
149  def InitQueue[T <: Data](entry: T, size: Int): Vec[T] =
150    RegInit(VecInit(Seq.fill(size)(0.U.asTypeOf(entry.cloneType))))
151
152  def getBankSel(blkOffset: UInt, valid: Bool = true.B): Vec[UInt] = {
153    val bankIdxLow  = (Cat(0.U(1.W), blkOffset) >> log2Ceil(blockBytes / ICacheDataBanks)).asUInt
154    val bankIdxHigh = ((Cat(0.U(1.W), blkOffset) + 32.U) >> log2Ceil(blockBytes / ICacheDataBanks)).asUInt
155    val bankSel     = VecInit((0 until ICacheDataBanks * 2).map(i => (i.U >= bankIdxLow) && (i.U <= bankIdxHigh)))
156    assert(
157      !valid || PopCount(bankSel) === ICacheBankVisitNum.U,
158      "The number of bank visits must be %d, but bankSel=0x%x",
159      ICacheBankVisitNum.U,
160      bankSel.asUInt
161    )
162    bankSel.asTypeOf(UInt((ICacheDataBanks * 2).W)).asTypeOf(Vec(2, UInt(ICacheDataBanks.W)))
163  }
164
165  def getLineSel(blkOffset: UInt): Vec[Bool] = {
166    val bankIdxLow = (blkOffset >> log2Ceil(blockBytes / ICacheDataBanks)).asUInt
167    val lineSel    = VecInit((0 until ICacheDataBanks).map(i => i.U < bankIdxLow))
168    lineSel
169  }
170
171  def getBlkAddr(addr:        UInt): UInt = (addr >> blockOffBits).asUInt
172  def getPhyTagFromBlk(addr:  UInt): UInt = (addr >> (pgUntagBits - blockOffBits)).asUInt
173  def getIdxFromBlk(addr:     UInt): UInt = addr(idxBits - 1, 0)
174  def getPaddrFromPtag(vaddr: UInt, ptag: UInt): UInt = Cat(ptag, vaddr(pgUntagBits - 1, 0))
175  def getPaddrFromPtag(vaddrVec: Vec[UInt], ptagVec: Vec[UInt]): Vec[UInt] =
176    VecInit((vaddrVec zip ptagVec).map { case (vaddr, ptag) => getPaddrFromPtag(vaddr, ptag) })
177}
178
179trait HasICacheECCHelper extends HasICacheParameters {
180  def encodeMetaECC(meta: UInt, poison: Bool = false.B): UInt = {
181    require(meta.getWidth == ICacheMetaBits)
182    val code = cacheParams.tagCode.encode(meta, poison) >> ICacheMetaBits
183    code.asTypeOf(UInt(ICacheMetaCodeBits.W))
184  }
185
186  def encodeDataECC(data: UInt, poison: Bool = false.B): UInt = {
187    require(data.getWidth == ICacheDataBits)
188    val datas = data.asTypeOf(Vec(ICacheDataCodeSegs, UInt((ICacheDataBits / ICacheDataCodeSegs).W)))
189    val codes = VecInit(datas.map(cacheParams.dataCode.encode(_, poison) >> (ICacheDataBits / ICacheDataCodeSegs)))
190    codes.asTypeOf(UInt(ICacheDataCodeBits.W))
191  }
192}
193
194abstract class ICacheBundle(implicit p: Parameters) extends XSBundle
195    with HasICacheParameters
196
197abstract class ICacheModule(implicit p: Parameters) extends XSModule
198    with HasICacheParameters
199
200abstract class ICacheArray(implicit p: Parameters) extends XSModule
201    with HasICacheParameters
202
203class ICacheMetadata(implicit p: Parameters) extends ICacheBundle {
204  val tag: UInt = UInt(tagBits.W)
205}
206
207object ICacheMetadata {
208  def apply(tag: Bits)(implicit p: Parameters): ICacheMetadata = {
209    val meta = Wire(new ICacheMetadata)
210    meta.tag := tag
211    meta
212  }
213}
214
215class ICacheMetaArrayIO(implicit p: Parameters) extends ICacheBundle {
216  val write:    DecoupledIO[ICacheMetaWriteBundle] = Flipped(DecoupledIO(new ICacheMetaWriteBundle))
217  val read:     DecoupledIO[ICacheReadBundle]      = Flipped(DecoupledIO(new ICacheReadBundle))
218  val readResp: ICacheMetaRespBundle               = Output(new ICacheMetaRespBundle)
219  val flush:    Vec[Valid[ICacheMetaFlushBundle]]  = Vec(PortNumber, Flipped(ValidIO(new ICacheMetaFlushBundle)))
220  val flushAll: Bool                               = Input(Bool())
221}
222
223class ICacheMetaArray(implicit p: Parameters) extends ICacheArray with HasICacheECCHelper {
224  class ICacheMetaEntry(implicit p: Parameters) extends ICacheBundle {
225    val meta: ICacheMetadata = new ICacheMetadata
226    val code: UInt           = UInt(ICacheMetaCodeBits.W)
227  }
228
229  private object ICacheMetaEntry {
230    def apply(meta: ICacheMetadata, poison: Bool)(implicit p: Parameters): ICacheMetaEntry = {
231      val entry = Wire(new ICacheMetaEntry)
232      entry.meta := meta
233      entry.code := encodeMetaECC(meta.asUInt, poison)
234      entry
235    }
236  }
237
238  // sanity check
239  require(ICacheMetaEntryBits == (new ICacheMetaEntry).getWidth)
240
241  val io: ICacheMetaArrayIO = IO(new ICacheMetaArrayIO)
242
243  private val port_0_read_0 = io.read.valid && !io.read.bits.vSetIdx(0)(0)
244  private val port_0_read_1 = io.read.valid && io.read.bits.vSetIdx(0)(0)
245  private val port_1_read_1 = io.read.valid && io.read.bits.vSetIdx(1)(0) && io.read.bits.isDoubleLine
246  private val port_1_read_0 = io.read.valid && !io.read.bits.vSetIdx(1)(0) && io.read.bits.isDoubleLine
247
248  private val port_0_read_0_reg = RegEnable(port_0_read_0, 0.U.asTypeOf(port_0_read_0), io.read.fire)
249  private val port_0_read_1_reg = RegEnable(port_0_read_1, 0.U.asTypeOf(port_0_read_1), io.read.fire)
250  private val port_1_read_1_reg = RegEnable(port_1_read_1, 0.U.asTypeOf(port_1_read_1), io.read.fire)
251  private val port_1_read_0_reg = RegEnable(port_1_read_0, 0.U.asTypeOf(port_1_read_0), io.read.fire)
252
253  private val bank_0_idx = Mux(port_0_read_0, io.read.bits.vSetIdx(0), io.read.bits.vSetIdx(1))
254  private val bank_1_idx = Mux(port_0_read_1, io.read.bits.vSetIdx(0), io.read.bits.vSetIdx(1))
255
256  private val write_bank_0 = io.write.valid && !io.write.bits.bankIdx
257  private val write_bank_1 = io.write.valid && io.write.bits.bankIdx
258
259  private val write_meta_bits = ICacheMetaEntry(
260    meta = ICacheMetadata(
261      tag = io.write.bits.phyTag
262    ),
263    poison = io.write.bits.poison
264  )
265
266  private val tagArrays = (0 until PortNumber) map { bank =>
267    val tagArray = Module(new SRAMTemplate(
268      new ICacheMetaEntry(),
269      set = nSets / PortNumber,
270      way = nWays,
271      shouldReset = true,
272      holdRead = true,
273      singlePort = true,
274      withClockGate = false // enable signal timing is bad, no gating here
275    ))
276
277    // meta connection
278    if (bank == 0) {
279      tagArray.io.r.req.valid := port_0_read_0 || port_1_read_0
280      tagArray.io.r.req.bits.apply(setIdx = bank_0_idx(highestIdxBit, 1))
281      tagArray.io.w.req.valid := write_bank_0
282      tagArray.io.w.req.bits.apply(
283        data = write_meta_bits,
284        setIdx = io.write.bits.virIdx(highestIdxBit, 1),
285        waymask = io.write.bits.waymask
286      )
287    } else {
288      tagArray.io.r.req.valid := port_0_read_1 || port_1_read_1
289      tagArray.io.r.req.bits.apply(setIdx = bank_1_idx(highestIdxBit, 1))
290      tagArray.io.w.req.valid := write_bank_1
291      tagArray.io.w.req.bits.apply(
292        data = write_meta_bits,
293        setIdx = io.write.bits.virIdx(highestIdxBit, 1),
294        waymask = io.write.bits.waymask
295      )
296    }
297
298    tagArray
299  }
300
301  private val read_set_idx_next = RegEnable(io.read.bits.vSetIdx, 0.U.asTypeOf(io.read.bits.vSetIdx), io.read.fire)
302  private val valid_array       = RegInit(VecInit(Seq.fill(nWays)(0.U(nSets.W))))
303  private val valid_metas       = Wire(Vec(PortNumber, Vec(nWays, Bool())))
304  // valid read
305  (0 until PortNumber).foreach(i =>
306    (0 until nWays).foreach(way =>
307      valid_metas(i)(way) := valid_array(way)(read_set_idx_next(i))
308    )
309  )
310  io.readResp.entryValid := valid_metas
311
312  io.read.ready := !io.write.valid && !io.flush.map(_.valid).reduce(_ || _) && !io.flushAll &&
313    tagArrays.map(_.io.r.req.ready).reduce(_ && _)
314
315  // valid write
316  private val way_num = OHToUInt(io.write.bits.waymask)
317  when(io.write.valid) {
318    valid_array(way_num) := valid_array(way_num).bitSet(io.write.bits.virIdx, true.B)
319  }
320
321  XSPerfAccumulate("meta_refill_num", io.write.valid)
322
323  io.readResp.metas <> DontCare
324  io.readResp.codes <> DontCare
325  private val readMetaEntries = tagArrays.map(port => port.io.r.resp.asTypeOf(Vec(nWays, new ICacheMetaEntry())))
326  private val readMetas       = readMetaEntries.map(_.map(_.meta))
327  private val readCodes       = readMetaEntries.map(_.map(_.code))
328
329  // TEST: force ECC to fail by setting readCodes to 0
330  if (ICacheForceMetaECCError) {
331    readCodes.foreach(_.foreach(_ := 0.U))
332  }
333
334  when(port_0_read_0_reg) {
335    io.readResp.metas(0) := readMetas(0)
336    io.readResp.codes(0) := readCodes(0)
337  }.elsewhen(port_0_read_1_reg) {
338    io.readResp.metas(0) := readMetas(1)
339    io.readResp.codes(0) := readCodes(1)
340  }
341
342  when(port_1_read_0_reg) {
343    io.readResp.metas(1) := readMetas(0)
344    io.readResp.codes(1) := readCodes(0)
345  }.elsewhen(port_1_read_1_reg) {
346    io.readResp.metas(1) := readMetas(1)
347    io.readResp.codes(1) := readCodes(1)
348  }
349
350  io.write.ready := true.B // TODO : has bug ? should be !io.cacheOp.req.valid
351
352  /*
353   * flush logic
354   */
355  // flush standalone set (e.g. flushed by mainPipe before doing re-fetch)
356  when(io.flush.map(_.valid).reduce(_ || _)) {
357    (0 until nWays).foreach { w =>
358      valid_array(w) := (0 until PortNumber).map { i =>
359        Mux(
360          // check if set `virIdx` in way `w` is requested to be flushed by port `i`
361          io.flush(i).valid && io.flush(i).bits.waymask(w),
362          valid_array(w).bitSet(io.flush(i).bits.virIdx, false.B),
363          valid_array(w)
364        )
365      }.reduce(_ & _)
366    }
367  }
368
369  // flush all (e.g. fence.i)
370  when(io.flushAll) {
371    (0 until nWays).foreach(w => valid_array(w) := 0.U)
372  }
373
374  // PERF: flush counter
375  XSPerfAccumulate("flush", io.flush.map(_.valid).reduce(_ || _))
376  XSPerfAccumulate("flush_all", io.flushAll)
377}
378
379class ICacheDataArrayIO(implicit p: Parameters) extends ICacheBundle {
380  val write:    DecoupledIO[ICacheDataWriteBundle] = Flipped(DecoupledIO(new ICacheDataWriteBundle))
381  val read:     Vec[DecoupledIO[ICacheReadBundle]] = Flipped(Vec(partWayNum, DecoupledIO(new ICacheReadBundle)))
382  val readResp: ICacheDataRespBundle               = Output(new ICacheDataRespBundle)
383}
384
385class ICacheDataArray(implicit p: Parameters) extends ICacheArray with HasICacheECCHelper {
386  class ICacheDataEntry(implicit p: Parameters) extends ICacheBundle {
387    val data: UInt = UInt(ICacheDataBits.W)
388    val code: UInt = UInt(ICacheDataCodeBits.W)
389  }
390
391  private object ICacheDataEntry {
392    def apply(data: UInt, poison: Bool)(implicit p: Parameters): ICacheDataEntry = {
393      val entry = Wire(new ICacheDataEntry)
394      entry.data := data
395      entry.code := encodeDataECC(data, poison)
396      entry
397    }
398  }
399
400  val io: ICacheDataArrayIO = IO(new ICacheDataArrayIO)
401
402  /**
403    ******************************************************************************
404    * data array
405    ******************************************************************************
406    */
407  private val writeDatas   = io.write.bits.data.asTypeOf(Vec(ICacheDataBanks, UInt(ICacheDataBits.W)))
408  private val writeEntries = writeDatas.map(ICacheDataEntry(_, io.write.bits.poison).asUInt)
409
410  // io.read() are copies to control fan-out, we can simply use .head here
411  private val bankSel  = getBankSel(io.read.head.bits.blkOffset, io.read.head.valid)
412  private val lineSel  = getLineSel(io.read.head.bits.blkOffset)
413  private val waymasks = io.read.head.bits.waymask
414  private val masks    = Wire(Vec(nWays, Vec(ICacheDataBanks, Bool())))
415  (0 until nWays).foreach { way =>
416    (0 until ICacheDataBanks).foreach { bank =>
417      masks(way)(bank) := Mux(
418        lineSel(bank),
419        waymasks(1)(way) && bankSel(1)(bank).asBool,
420        waymasks(0)(way) && bankSel(0)(bank).asBool
421      )
422    }
423  }
424
425  private val dataArrays = (0 until nWays).map { way =>
426    (0 until ICacheDataBanks).map { bank =>
427      val sramBank = Module(new SRAMTemplateWithFixedWidth(
428        UInt(ICacheDataEntryBits.W),
429        set = nSets,
430        width = ICacheDataSRAMWidth,
431        shouldReset = true,
432        holdRead = true,
433        singlePort = true,
434        withClockGate = true
435      ))
436
437      // read
438      sramBank.io.r.req.valid := io.read(bank % 4).valid && masks(way)(bank)
439      sramBank.io.r.req.bits.apply(setIdx =
440        Mux(lineSel(bank), io.read(bank % 4).bits.vSetIdx(1), io.read(bank % 4).bits.vSetIdx(0))
441      )
442      // write
443      sramBank.io.w.req.valid := io.write.valid && io.write.bits.waymask(way).asBool
444      sramBank.io.w.req.bits.apply(
445        data = writeEntries(bank),
446        setIdx = io.write.bits.virIdx,
447        // waymask is invalid when way of SRAMTemplate <= 1
448        waymask = 0.U
449      )
450      sramBank
451    }
452  }
453
454  /**
455    ******************************************************************************
456    * read logic
457    ******************************************************************************
458    */
459  private val masksReg = RegEnable(masks, 0.U.asTypeOf(masks), io.read(0).valid)
460  private val readDataWithCode = (0 until ICacheDataBanks).map { bank =>
461    Mux1H(VecInit(masksReg.map(_(bank))).asTypeOf(UInt(nWays.W)), dataArrays.map(_(bank).io.r.resp.asUInt))
462  }
463  private val readEntries = readDataWithCode.map(_.asTypeOf(new ICacheDataEntry()))
464  private val readDatas   = VecInit(readEntries.map(_.data))
465  private val readCodes   = VecInit(readEntries.map(_.code))
466
467  // TEST: force ECC to fail by setting readCodes to 0
468  if (ICacheForceDataECCError) {
469    readCodes.foreach(_ := 0.U)
470  }
471
472  /**
473    ******************************************************************************
474    * IO
475    ******************************************************************************
476    */
477  io.readResp.datas := readDatas
478  io.readResp.codes := readCodes
479  io.write.ready    := true.B
480  io.read.foreach(_.ready := !io.write.valid)
481}
482
483class ICacheReplacerIO(implicit p: Parameters) extends ICacheBundle {
484  val touch:  Vec[Valid[ReplacerTouch]] = Vec(PortNumber, Flipped(ValidIO(new ReplacerTouch)))
485  val victim: ReplacerVictim            = Flipped(new ReplacerVictim)
486}
487
488class ICacheReplacer(implicit p: Parameters) extends ICacheModule {
489  val io: ICacheReplacerIO = IO(new ICacheReplacerIO)
490
491  private val replacers =
492    Seq.fill(PortNumber)(ReplacementPolicy.fromString(cacheParams.replacer, nWays, nSets / PortNumber))
493
494  // touch
495  private val touch_sets = Seq.fill(PortNumber)(Wire(Vec(PortNumber, UInt(log2Ceil(nSets / PortNumber).W))))
496  private val touch_ways = Seq.fill(PortNumber)(Wire(Vec(PortNumber, Valid(UInt(wayBits.W)))))
497  (0 until PortNumber).foreach { i =>
498    touch_sets(i)(0) := Mux(
499      io.touch(i).bits.vSetIdx(0),
500      io.touch(1).bits.vSetIdx(highestIdxBit, 1),
501      io.touch(0).bits.vSetIdx(highestIdxBit, 1)
502    )
503    touch_ways(i)(0).bits  := Mux(io.touch(i).bits.vSetIdx(0), io.touch(1).bits.way, io.touch(0).bits.way)
504    touch_ways(i)(0).valid := Mux(io.touch(i).bits.vSetIdx(0), io.touch(1).valid, io.touch(0).valid)
505  }
506
507  // victim
508  io.victim.way := Mux(
509    io.victim.vSetIdx.bits(0),
510    replacers(1).way(io.victim.vSetIdx.bits(highestIdxBit, 1)),
511    replacers(0).way(io.victim.vSetIdx.bits(highestIdxBit, 1))
512  )
513
514  // touch the victim in next cycle
515  private val victim_vSetIdx_reg =
516    RegEnable(io.victim.vSetIdx.bits, 0.U.asTypeOf(io.victim.vSetIdx.bits), io.victim.vSetIdx.valid)
517  private val victim_way_reg = RegEnable(io.victim.way, 0.U.asTypeOf(io.victim.way), io.victim.vSetIdx.valid)
518  (0 until PortNumber).foreach { i =>
519    touch_sets(i)(1)       := victim_vSetIdx_reg(highestIdxBit, 1)
520    touch_ways(i)(1).bits  := victim_way_reg
521    touch_ways(i)(1).valid := RegNext(io.victim.vSetIdx.valid) && (victim_vSetIdx_reg(0) === i.U)
522  }
523
524  ((replacers zip touch_sets) zip touch_ways).foreach { case ((r, s), w) => r.access(s, w) }
525}
526
527class ICacheIO(implicit p: Parameters) extends ICacheBundle {
528  val hartId: UInt = Input(UInt(hartIdLen.W))
529  // FTQ
530  val fetch:       ICacheMainPipeBundle = new ICacheMainPipeBundle
531  val ftqPrefetch: FtqToPrefetchIO      = Flipped(new FtqToPrefetchIO)
532  // memblock
533  val softPrefetch: Vec[Valid[SoftIfetchPrefetchBundle]] =
534    Vec(backendParams.LduCnt, Flipped(Valid(new SoftIfetchPrefetchBundle)))
535  // IFU
536  val stop:  Bool = Input(Bool())
537  val toIFU: Bool = Output(Bool())
538  // PMP: mainPipe & prefetchPipe need PortNumber each
539  val pmp: Vec[ICachePMPBundle] = Vec(2 * PortNumber, new ICachePMPBundle)
540  // iTLB
541  val itlb:          Vec[TlbRequestIO] = Vec(PortNumber, new TlbRequestIO)
542  val itlbFlushPipe: Bool              = Bool()
543  // backend/BEU
544  val error: Valid[L1CacheErrorInfo] = ValidIO(new L1CacheErrorInfo)
545  // backend/CSR
546  val csr_pf_enable: Bool = Input(Bool())
547  // flush
548  val fencei: Bool = Input(Bool())
549  val flush:  Bool = Input(Bool())
550
551  // perf
552  val perfInfo: ICachePerfInfo = Output(new ICachePerfInfo)
553}
554
555class ICache()(implicit p: Parameters) extends LazyModule with HasICacheParameters {
556  override def shouldBeInlined: Boolean = false
557
558  val clientParameters: TLMasterPortParameters = TLMasterPortParameters.v1(
559    Seq(TLMasterParameters.v1(
560      name = "icache",
561      sourceId = IdRange(0, cacheParams.nFetchMshr + cacheParams.nPrefetchMshr + 1)
562    )),
563    requestFields = cacheParams.reqFields,
564    echoFields = cacheParams.echoFields
565  )
566
567  val clientNode: TLClientNode = TLClientNode(Seq(clientParameters))
568
569  val ctrlUnitOpt: Option[ICacheCtrlUnit] = ctrlUnitParamsOpt.map(params => LazyModule(new ICacheCtrlUnit(params)))
570
571  lazy val module: ICacheImp = new ICacheImp(this)
572}
573
574class ICacheImp(outer: ICache) extends LazyModuleImp(outer) with HasICacheParameters with HasPerfEvents {
575  val io: ICacheIO = IO(new ICacheIO)
576
577  println("ICache:")
578  println("  TagECC: " + cacheParams.tagECC)
579  println("  DataECC: " + cacheParams.dataECC)
580  println("  ICacheSets: " + cacheParams.nSets)
581  println("  ICacheWays: " + cacheParams.nWays)
582  println("  PortNumber: " + cacheParams.PortNumber)
583  println("  nFetchMshr: " + cacheParams.nFetchMshr)
584  println("  nPrefetchMshr: " + cacheParams.nPrefetchMshr)
585  println("  nWayLookupSize: " + cacheParams.nWayLookupSize)
586  println("  DataCodeUnit: " + cacheParams.DataCodeUnit)
587  println("  ICacheDataBanks: " + cacheParams.ICacheDataBanks)
588  println("  ICacheDataSRAMWidth: " + cacheParams.ICacheDataSRAMWidth)
589
590  val (bus, edge) = outer.clientNode.out.head
591
592  private val metaArray  = Module(new ICacheMetaArray)
593  private val dataArray  = Module(new ICacheDataArray)
594  private val mainPipe   = Module(new ICacheMainPipe)
595  private val missUnit   = Module(new ICacheMissUnit(edge))
596  private val replacer   = Module(new ICacheReplacer)
597  private val prefetcher = Module(new IPrefetchPipe)
598  private val wayLookup  = Module(new WayLookup)
599
600  private val ecc_enable = if (outer.ctrlUnitOpt.nonEmpty) outer.ctrlUnitOpt.get.module.io.ecc_enable else true.B
601
602  // dataArray io
603  if (outer.ctrlUnitOpt.nonEmpty) {
604    val ctrlUnit = outer.ctrlUnitOpt.get.module
605    when(ctrlUnit.io.injecting) {
606      dataArray.io.write <> ctrlUnit.io.dataWrite
607      missUnit.io.data_write.ready := false.B
608    }.otherwise {
609      ctrlUnit.io.dataWrite.ready := false.B
610      dataArray.io.write <> missUnit.io.data_write
611    }
612  } else {
613    dataArray.io.write <> missUnit.io.data_write
614  }
615  dataArray.io.read <> mainPipe.io.dataArray.toIData
616  mainPipe.io.dataArray.fromIData := dataArray.io.readResp
617
618  // metaArray io
619  metaArray.io.flushAll := io.fencei
620  metaArray.io.flush <> mainPipe.io.metaArrayFlush
621  if (outer.ctrlUnitOpt.nonEmpty) {
622    val ctrlUnit = outer.ctrlUnitOpt.get.module
623    when(ctrlUnit.io.injecting) {
624      metaArray.io.write <> ctrlUnit.io.metaWrite
625      metaArray.io.read <> ctrlUnit.io.metaRead
626      missUnit.io.meta_write.ready         := false.B
627      prefetcher.io.metaRead.toIMeta.ready := false.B
628    }.otherwise {
629      ctrlUnit.io.metaWrite.ready := false.B
630      ctrlUnit.io.metaRead.ready  := false.B
631      metaArray.io.write <> missUnit.io.meta_write
632      metaArray.io.read <> prefetcher.io.metaRead.toIMeta
633    }
634    ctrlUnit.io.metaReadResp := metaArray.io.readResp
635  } else {
636    metaArray.io.write <> missUnit.io.meta_write
637    metaArray.io.read <> prefetcher.io.metaRead.toIMeta
638  }
639  prefetcher.io.metaRead.fromIMeta := metaArray.io.readResp
640
641  prefetcher.io.flush         := io.flush
642  prefetcher.io.csr_pf_enable := io.csr_pf_enable
643  prefetcher.io.ecc_enable    := ecc_enable
644  prefetcher.io.MSHRResp      := missUnit.io.fetch_resp
645  prefetcher.io.flushFromBpu  := io.ftqPrefetch.flushFromBpu
646  // cache softPrefetch
647  private val softPrefetchValid = RegInit(false.B)
648  private val softPrefetch      = RegInit(0.U.asTypeOf(new IPrefetchReq))
649  /* FIXME:
650   * If there is already a pending softPrefetch request, it will be overwritten.
651   * Also, if there are multiple softPrefetch requests in the same cycle, only the first one will be accepted.
652   * We should implement a softPrefetchQueue (like ibuffer, multi-in, single-out) to solve this.
653   * However, the impact on performance still needs to be assessed.
654   * Considering that the frequency of prefetch.i may not be high, let's start with a temporary dummy solution.
655   */
656  when(io.softPrefetch.map(_.valid).reduce(_ || _)) {
657    softPrefetchValid := true.B
658    softPrefetch.fromSoftPrefetch(MuxCase(
659      0.U.asTypeOf(new SoftIfetchPrefetchBundle),
660      io.softPrefetch.map(req => req.valid -> req.bits)
661    ))
662  }.elsewhen(prefetcher.io.req.fire) {
663    softPrefetchValid := false.B
664  }
665  // pass ftqPrefetch
666  private val ftqPrefetch = WireInit(0.U.asTypeOf(new IPrefetchReq))
667  ftqPrefetch.fromFtqICacheInfo(io.ftqPrefetch.req.bits)
668  // software prefetch has higher priority
669  prefetcher.io.req.valid                 := softPrefetchValid || io.ftqPrefetch.req.valid
670  prefetcher.io.req.bits                  := Mux(softPrefetchValid, softPrefetch, ftqPrefetch)
671  prefetcher.io.req.bits.backendException := io.ftqPrefetch.backendException
672  io.ftqPrefetch.req.ready                := prefetcher.io.req.ready && !softPrefetchValid
673
674  missUnit.io.hartId := io.hartId
675  missUnit.io.fencei := io.fencei
676  missUnit.io.flush  := io.flush
677  missUnit.io.fetch_req <> mainPipe.io.mshr.req
678  missUnit.io.prefetch_req <> prefetcher.io.MSHRReq
679  missUnit.io.mem_grant.valid := false.B
680  missUnit.io.mem_grant.bits  := DontCare
681  missUnit.io.mem_grant <> bus.d
682
683  mainPipe.io.flush      := io.flush
684  mainPipe.io.respStall  := io.stop
685  mainPipe.io.ecc_enable := ecc_enable
686  mainPipe.io.hartId     := io.hartId
687  mainPipe.io.mshr.resp  := missUnit.io.fetch_resp
688  mainPipe.io.fetch.req <> io.fetch.req
689  mainPipe.io.wayLookupRead <> wayLookup.io.read
690
691  wayLookup.io.flush := io.flush
692  wayLookup.io.write <> prefetcher.io.wayLookupWrite
693  wayLookup.io.update := missUnit.io.fetch_resp
694
695  replacer.io.touch <> mainPipe.io.touch
696  replacer.io.victim <> missUnit.io.victim
697
698  io.pmp(0) <> mainPipe.io.pmp(0)
699  io.pmp(1) <> mainPipe.io.pmp(1)
700  io.pmp(2) <> prefetcher.io.pmp(0)
701  io.pmp(3) <> prefetcher.io.pmp(1)
702
703  io.itlb(0) <> prefetcher.io.itlb(0)
704  io.itlb(1) <> prefetcher.io.itlb(1)
705  io.itlbFlushPipe := prefetcher.io.itlbFlushPipe
706
707  // notify IFU that Icache pipeline is available
708  io.toIFU    := mainPipe.io.fetch.req.ready
709  io.perfInfo := mainPipe.io.perfInfo
710
711  io.fetch.resp <> mainPipe.io.fetch.resp
712  io.fetch.topdownIcacheMiss := mainPipe.io.fetch.topdownIcacheMiss
713  io.fetch.topdownItlbMiss   := mainPipe.io.fetch.topdownItlbMiss
714
715  bus.b.ready := false.B
716  bus.c.valid := false.B
717  bus.c.bits  := DontCare
718  bus.e.valid := false.B
719  bus.e.bits  := DontCare
720
721  bus.a <> missUnit.io.mem_acquire
722
723  // Parity error port
724  private val errors       = mainPipe.io.errors
725  private val errors_valid = errors.map(e => e.valid).reduce(_ | _)
726  io.error.bits <> RegEnable(
727    PriorityMux(errors.map(e => e.valid -> e.bits)),
728    0.U.asTypeOf(errors(0).bits),
729    errors_valid
730  )
731  io.error.valid := RegNext(errors_valid, false.B)
732
733  XSPerfAccumulate(
734    "softPrefetch_drop_not_ready",
735    io.softPrefetch.map(_.valid).reduce(_ || _) && softPrefetchValid && !prefetcher.io.req.fire
736  )
737  XSPerfAccumulate("softPrefetch_drop_multi_req", PopCount(io.softPrefetch.map(_.valid)) > 1.U)
738  XSPerfAccumulate("softPrefetch_block_ftq", softPrefetchValid && io.ftqPrefetch.req.valid)
739
740  val perfEvents: Seq[(String, Bool)] = Seq(
741    ("icache_miss_cnt  ", false.B),
742    ("icache_miss_penalty", BoolStopWatch(start = false.B, stop = false.B || false.B, startHighPriority = true))
743  )
744  generatePerfEvent()
745}
746
747//class ICachePartWayReadBundle[T <: Data](gen: T, pWay: Int)(implicit p: Parameters)
748//    extends ICacheBundle {
749//  val req = Flipped(Vec(
750//    PortNumber,
751//    Decoupled(new Bundle {
752//      val ridx = UInt((log2Ceil(nSets) - 1).W)
753//    })
754//  ))
755//  val resp = Output(new Bundle {
756//    val rdata = Vec(PortNumber, Vec(pWay, gen))
757//  })
758//}
759
760//class ICacheWriteBundle[T <: Data](gen: T, pWay: Int)(implicit p: Parameters)
761//    extends ICacheBundle {
762//  val wdata    = gen
763//  val widx     = UInt((log2Ceil(nSets) - 1).W)
764//  val wbankidx = Bool()
765//  val wmask    = Vec(pWay, Bool())
766//}
767
768//class ICachePartWayArray[T <: Data](gen: T, pWay: Int)(implicit p: Parameters) extends ICacheArray {
769//
770//  // including part way data
771//  val io = IO {
772//    new Bundle {
773//      val read  = new ICachePartWayReadBundle(gen, pWay)
774//      val write = Flipped(ValidIO(new ICacheWriteBundle(gen, pWay)))
775//    }
776//  }
777//
778//  io.read.req.map(_.ready := !io.write.valid)
779//
780//  val srams = (0 until PortNumber) map { bank =>
781//    val sramBank = Module(new SRAMTemplate(
782//      gen,
783//      set = nSets / 2,
784//      way = pWay,
785//      shouldReset = true,
786//      holdRead = true,
787//      singlePort = true,
788//      withClockGate = true
789//    ))
790//
791//    sramBank.io.r.req.valid := io.read.req(bank).valid
792//    sramBank.io.r.req.bits.apply(setIdx = io.read.req(bank).bits.ridx)
793//
794//    if (bank == 0) sramBank.io.w.req.valid := io.write.valid && !io.write.bits.wbankidx
795//    else sramBank.io.w.req.valid           := io.write.valid && io.write.bits.wbankidx
796//    sramBank.io.w.req.bits.apply(
797//      data = io.write.bits.wdata,
798//      setIdx = io.write.bits.widx,
799//      waymask = io.write.bits.wmask.asUInt
800//    )
801//
802//    sramBank
803//  }
804//
805//  io.read.req.map(_.ready := !io.write.valid && srams.map(_.io.r.req.ready).reduce(_ && _))
806//
807//  io.read.resp.rdata := VecInit(srams.map(bank => bank.io.r.resp.asTypeOf(Vec(pWay, gen))))
808//
809//}
810
811class SRAMTemplateWithFixedWidthIO[T <: Data](gen: T, set: Int, way: Int) extends Bundle {
812  val r: SRAMReadBus[T]  = Flipped(new SRAMReadBus(gen, set, way))
813  val w: SRAMWriteBus[T] = Flipped(new SRAMWriteBus(gen, set, way))
814}
815
816// Automatically partition the SRAM based on the width of the data and the desired width.
817// final SRAM width = width * way
818class SRAMTemplateWithFixedWidth[T <: Data](
819    gen:           T,
820    set:           Int,
821    width:         Int,
822    way:           Int = 1,
823    shouldReset:   Boolean = false,
824    holdRead:      Boolean = false,
825    singlePort:    Boolean = false,
826    bypassWrite:   Boolean = false,
827    withClockGate: Boolean = false
828) extends Module {
829
830  private val dataBits  = gen.getWidth
831  private val bankNum   = math.ceil(dataBits.toDouble / width.toDouble).toInt
832  private val totalBits = bankNum * width
833
834  val io: SRAMTemplateWithFixedWidthIO[T] = IO(new SRAMTemplateWithFixedWidthIO(gen, set, way))
835
836  private val wordType = UInt(width.W)
837  private val writeDatas = (0 until bankNum).map { bank =>
838    VecInit((0 until way).map { i =>
839      io.w.req.bits.data(i).asTypeOf(UInt(totalBits.W)).asTypeOf(Vec(bankNum, wordType))(bank)
840    })
841  }
842
843  private val srams = (0 until bankNum) map { bank =>
844    val sramBank = Module(new SRAMTemplate(
845      wordType,
846      set = set,
847      way = way,
848      shouldReset = shouldReset,
849      holdRead = holdRead,
850      singlePort = singlePort,
851      bypassWrite = bypassWrite,
852      withClockGate = withClockGate
853    ))
854    // read req
855    sramBank.io.r.req.valid       := io.r.req.valid
856    sramBank.io.r.req.bits.setIdx := io.r.req.bits.setIdx
857
858    // write req
859    sramBank.io.w.req.valid       := io.w.req.valid
860    sramBank.io.w.req.bits.setIdx := io.w.req.bits.setIdx
861    sramBank.io.w.req.bits.data   := writeDatas(bank)
862    sramBank.io.w.req.bits.waymask.foreach(_ := io.w.req.bits.waymask.get)
863
864    sramBank
865  }
866
867  io.r.req.ready := !io.w.req.valid
868  (0 until way).foreach { i =>
869    io.r.resp.data(i) := VecInit((0 until bankNum).map(bank =>
870      srams(bank).io.r.resp.data(i)
871    )).asTypeOf(UInt(totalBits.W))(dataBits - 1, 0).asTypeOf(gen.cloneType)
872  }
873
874  io.r.req.ready := srams.head.io.r.req.ready
875  io.w.req.ready := srams.head.io.w.req.ready
876}
877