1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.cache 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import utils._ 22import utility._ 23import chisel3.util._ 24import freechips.rocketchip.tilelink.{ClientMetadata, TLClientParameters, TLEdgeOut} 25import xiangshan.{L1CacheErrorInfo, XSCoreParamsKey} 26 27import scala.math.max 28 29class BankConflictDB(implicit p: Parameters) extends DCacheBundle{ 30 val addr = Vec(LoadPipelineWidth, Bits(PAddrBits.W)) 31 val set_index = Vec(LoadPipelineWidth, UInt((DCacheAboveIndexOffset - DCacheSetOffset).W)) 32 val bank_index = Vec(VLEN/DCacheSRAMRowBits, UInt((DCacheSetOffset - DCacheBankOffset).W)) 33 val way_index = UInt(wayBits.W) 34 val fake_rr_bank_conflict = Bool() 35} 36 37class L1BankedDataReadReq(implicit p: Parameters) extends DCacheBundle 38{ 39 val way_en = Bits(DCacheWays.W) 40 val addr = Bits(PAddrBits.W) 41} 42 43class L1BankedDataReadReqWithMask(implicit p: Parameters) extends DCacheBundle 44{ 45 val way_en = Bits(DCacheWays.W) 46 val addr = Bits(PAddrBits.W) 47 val bankMask = Bits(DCacheBanks.W) 48} 49 50class L1BankedDataReadLineReq(implicit p: Parameters) extends L1BankedDataReadReq 51{ 52 val rmask = Bits(DCacheBanks.W) 53} 54 55// Now, we can write a cache-block in a single cycle 56class L1BankedDataWriteReq(implicit p: Parameters) extends L1BankedDataReadReq 57{ 58 val wmask = Bits(DCacheBanks.W) 59 val data = Vec(DCacheBanks, Bits(DCacheSRAMRowBits.W)) 60} 61 62// cache-block write request without data 63class L1BankedDataWriteReqCtrl(implicit p: Parameters) extends L1BankedDataReadReq 64 65class L1BankedDataReadResult(implicit p: Parameters) extends DCacheBundle 66{ 67 // you can choose which bank to read to save power 68 val ecc = Bits(eccBits.W) 69 val raw_data = Bits(DCacheSRAMRowBits.W) 70 val error_delayed = Bool() // 1 cycle later than data resp 71 72 def asECCData() = { 73 Cat(ecc, raw_data) 74 } 75} 76 77class DataSRAMBankWriteReq(implicit p: Parameters) extends DCacheBundle { 78 val en = Bool() 79 val addr = UInt() 80 val way_en = UInt(DCacheWays.W) 81 val data = UInt(DCacheSRAMRowBits.W) 82} 83 84// wrap a sram 85class DataSRAM(bankIdx: Int, wayIdx: Int)(implicit p: Parameters) extends DCacheModule { 86 val io = IO(new Bundle() { 87 val w = new Bundle() { 88 val en = Input(Bool()) 89 val addr = Input(UInt()) 90 val data = Input(UInt(DCacheSRAMRowBits.W)) 91 } 92 93 val r = new Bundle() { 94 val en = Input(Bool()) 95 val addr = Input(UInt()) 96 val data = Output(UInt(DCacheSRAMRowBits.W)) 97 } 98 }) 99 100 // data sram 101 val data_sram = Module(new SRAMTemplate( 102 Bits(DCacheSRAMRowBits.W), 103 set = DCacheSets / DCacheSetDiv, 104 way = 1, 105 shouldReset = false, 106 holdRead = false, 107 singlePort = true 108 )) 109 110 data_sram.io.w.req.valid := io.w.en 111 data_sram.io.w.req.bits.apply( 112 setIdx = io.w.addr, 113 data = io.w.data, 114 waymask = 1.U 115 ) 116 data_sram.io.r.req.valid := io.r.en 117 data_sram.io.r.req.bits.apply(setIdx = io.r.addr) 118 io.r.data := data_sram.io.r.resp.data(0) 119 XSPerfAccumulate("part_data_read_counter", data_sram.io.r.req.valid) 120 121 def dump_r() = { 122 when(RegNext(io.r.en)) { 123 XSDebug("bank read set %x bank %x way %x data %x\n", 124 RegNext(io.r.addr), 125 bankIdx.U, 126 wayIdx.U, 127 io.r.data 128 ) 129 } 130 } 131 132 def dump_w() = { 133 when(io.w.en) { 134 XSDebug("bank write set %x bank %x way %x data %x\n", 135 io.w.addr, 136 bankIdx.U, 137 wayIdx.U, 138 io.w.data 139 ) 140 } 141 } 142 143 def dump() = { 144 dump_w() 145 dump_r() 146 } 147} 148 149// wrap data rows of 8 ways 150class DataSRAMBank(index: Int)(implicit p: Parameters) extends DCacheModule { 151 val io = IO(new Bundle() { 152 val w = Input(new DataSRAMBankWriteReq) 153 154 val r = new Bundle() { 155 val en = Input(Bool()) 156 val addr = Input(UInt()) 157 val way_en = Input(UInt(DCacheWays.W)) 158 val data = Output(UInt(DCacheSRAMRowBits.W)) 159 } 160 }) 161 162 assert(RegNext(!io.w.en || PopCount(io.w.way_en) <= 1.U)) 163 assert(RegNext(!io.r.en || PopCount(io.r.way_en) <= 1.U)) 164 165 val r_way_en_reg = RegEnable(io.r.way_en, io.r.en) 166 167 // external controls do not read and write at the same time 168 val w_info = io.w 169 // val rw_bypass = RegNext(io.w.addr === io.r.addr && io.w.way_en === io.r.way_en && io.w.en) 170 171 // multiway data bank 172 val data_bank = Array.fill(DCacheWays) { 173 Module(new SRAMTemplate( 174 Bits(DCacheSRAMRowBits.W), 175 set = DCacheSets / DCacheSetDiv, 176 way = 1, 177 shouldReset = false, 178 holdRead = false, 179 singlePort = true 180 )) 181 } 182 183 for (w <- 0 until DCacheWays) { 184 val wen = w_info.en && w_info.way_en(w) 185 data_bank(w).io.w.req.valid := wen 186 data_bank(w).io.w.req.bits.apply( 187 setIdx = w_info.addr, 188 data = w_info.data, 189 waymask = 1.U 190 ) 191 data_bank(w).io.r.req.valid := io.r.en 192 data_bank(w).io.r.req.bits.apply(setIdx = io.r.addr) 193 } 194 XSPerfAccumulate("part_data_read_counter", PopCount(Cat(data_bank.map(_.io.r.req.valid)))) 195 196 val half = nWays / 2 197 val data_read = data_bank.map(_.io.r.resp.data(0)) 198 val data_left = Mux1H(r_way_en_reg.tail(half), data_read.take(half)) 199 val data_right = Mux1H(r_way_en_reg.head(half), data_read.drop(half)) 200 201 val sel_low = r_way_en_reg.tail(half).orR 202 val row_data = Mux(sel_low, data_left, data_right) 203 204 io.r.data := row_data 205 206 def dump_r() = { 207 when(RegNext(io.r.en)) { 208 XSDebug("bank read addr %x way_en %x data %x\n", 209 RegNext(io.r.addr), 210 RegNext(io.r.way_en), 211 io.r.data 212 ) 213 } 214 } 215 216 def dump_w() = { 217 when(io.w.en) { 218 XSDebug("bank write addr %x way_en %x data %x\n", 219 io.w.addr, 220 io.w.way_en, 221 io.w.data 222 ) 223 } 224 } 225 226 def dump() = { 227 dump_w() 228 dump_r() 229 } 230} 231 232case object HasDataEccParam 233 234// Banked DCache Data 235// ----------------------------------------------------------------- 236// | Bank0 | Bank1 | Bank2 | Bank3 | Bank4 | Bank5 | Bank6 | Bank7 | 237// ----------------------------------------------------------------- 238// | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | Way0 | 239// | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | Way1 | 240// | .... | .... | .... | .... | .... | .... | .... | .... | 241// ----------------------------------------------------------------- 242abstract class AbstractBankedDataArray(implicit p: Parameters) extends DCacheModule 243{ 244 val DataEccParam = if(EnableDataEcc) Some(HasDataEccParam) else None 245 val ReadlinePortErrorIndex = LoadPipelineWidth 246 val io = IO(new DCacheBundle { 247 // load pipeline read word req 248 val read = Vec(LoadPipelineWidth, Flipped(DecoupledIO(new L1BankedDataReadReqWithMask))) 249 val is128Req = Input(Vec(LoadPipelineWidth, Bool())) 250 // main pipeline read / write line req 251 val readline_intend = Input(Bool()) 252 val readline = Flipped(DecoupledIO(new L1BankedDataReadLineReq)) 253 val write = Flipped(DecoupledIO(new L1BankedDataWriteReq)) 254 val write_dup = Vec(DCacheBanks, Flipped(Decoupled(new L1BankedDataWriteReqCtrl))) 255 // data for readline and loadpipe 256 val readline_resp = Output(Vec(DCacheBanks, new L1BankedDataReadResult())) 257 val readline_error_delayed = Output(Bool()) 258 val read_resp_delayed = Output(Vec(LoadPipelineWidth, Vec(VLEN/DCacheSRAMRowBits, new L1BankedDataReadResult()))) 259 val read_error_delayed = Output(Vec(LoadPipelineWidth,Vec(VLEN/DCacheSRAMRowBits, Bool()))) 260 // val nacks = Output(Vec(LoadPipelineWidth, Bool())) 261 // val errors = Output(Vec(LoadPipelineWidth + 1, ValidIO(new L1CacheErrorInfo))) // read ports + readline port 262 // when bank_conflict, read (1) port should be ignored 263 val bank_conflict_slow = Output(Vec(LoadPipelineWidth, Bool())) 264 val disable_ld_fast_wakeup = Output(Vec(LoadPipelineWidth, Bool())) 265 // customized cache op port 266 val cacheOp = Flipped(new L1CacheInnerOpIO) 267 val cacheOp_req_dup = Vec(DCacheDupNum, Flipped(Valid(new CacheCtrlReqInfo))) 268 val cacheOp_req_bits_opCode_dup = Input(Vec(DCacheDupNum, UInt(XLEN.W))) 269 }) 270 271 def pipeMap[T <: Data](f: Int => T) = VecInit((0 until LoadPipelineWidth).map(f)) 272 273 def getECCFromEncWord(encWord: UInt) = { 274 require(encWord.getWidth == encWordBits) 275 encWord(encWordBits - 1, wordBits) 276 } 277 278 def dumpRead() = { 279 (0 until LoadPipelineWidth) map { w => 280 when(io.read(w).valid) { 281 XSDebug(s"DataArray Read channel: $w valid way_en: %x addr: %x\n", 282 io.read(w).bits.way_en, io.read(w).bits.addr) 283 } 284 } 285 when(io.readline.valid) { 286 XSDebug(s"DataArray Read Line, valid way_en: %x addr: %x rmask %x\n", 287 io.readline.bits.way_en, io.readline.bits.addr, io.readline.bits.rmask) 288 } 289 } 290 291 def dumpWrite() = { 292 when(io.write.valid) { 293 XSDebug(s"DataArray Write valid way_en: %x addr: %x\n", 294 io.write.bits.way_en, io.write.bits.addr) 295 296 (0 until DCacheBanks) map { r => 297 XSDebug(s"cycle: $r data: %x wmask: %x\n", 298 io.write.bits.data(r), io.write.bits.wmask(r)) 299 } 300 } 301 } 302 303 def dumpResp() = { 304 XSDebug(s"DataArray ReadeResp channel:\n") 305 (0 until LoadPipelineWidth) map { r => 306 XSDebug(s"cycle: $r data: %x\n", Mux(io.is128Req(r), 307 Cat(io.read_resp_delayed(r)(1).raw_data,io.read_resp_delayed(r)(0).raw_data), 308 io.read_resp_delayed(r)(0).raw_data)) 309 } 310 } 311 312 def dump() = { 313 dumpRead 314 dumpWrite 315 dumpResp 316 } 317} 318 319// the smallest access unit is sram 320class SramedDataArray(implicit p: Parameters) extends AbstractBankedDataArray { 321 println(" DCacheType: SramedDataArray") 322 val ReduceReadlineConflict = false 323 324 io.write.ready := true.B 325 io.write_dup.foreach(_.ready := true.B) 326 327 val data_banks = List.tabulate(DCacheSetDiv)( k => List.tabulate(DCacheBanks)(i => List.tabulate(DCacheWays)(j => Module(new DataSRAM(i,j))))) 328 // ecc_banks also needs to be changed to two-dimensional to align with data_banks 329 val ecc_banks = DataEccParam.map { 330 case _ => 331 val ecc = List.tabulate(DCacheSetDiv)( k => 332 List.tabulate(DCacheWays)(j => 333 List.tabulate(DCacheBanks)(i => 334 Module(new SRAMTemplate( 335 Bits(eccBits.W), 336 set = DCacheSets / DCacheSetDiv, 337 way = 1, 338 shouldReset = false, 339 holdRead = false, 340 singlePort = true 341 )) 342 ))) 343 ecc 344 } 345 346 data_banks.map(_.map(_.map(_.dump()))) 347 348 val way_en = Wire(Vec(LoadPipelineWidth, io.read(0).bits.way_en.cloneType)) 349 val set_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 350 val div_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 351 val bank_addrs = Wire(Vec(LoadPipelineWidth, Vec(VLEN/DCacheSRAMRowBits, UInt()))) 352 353 val line_set_addr = addr_to_dcache_div_set(io.readline.bits.addr) 354 val line_div_addr = addr_to_dcache_div(io.readline.bits.addr) 355 // when WPU is enabled, line_way_en is all enabled when read data 356 val line_way_en = Fill(DCacheWays, 1.U) // val line_way_en = io.readline.bits.way_en 357 val line_way_en_reg = RegEnable(io.readline.bits.way_en, io.readline.valid) 358 359 val write_bank_mask_reg = RegEnable(io.write.bits.wmask, io.write.valid) 360 val write_data_reg = RegEnable(io.write.bits.data, io.write.valid) 361 val write_valid_reg = RegNext(io.write.valid) 362 val write_valid_dup_reg = io.write_dup.map(x => RegNext(x.valid)) 363 val write_wayen_dup_reg = io.write_dup.map(x => RegEnable(x.bits.way_en, x.valid)) 364 val write_set_addr_dup_reg = io.write_dup.map(x => RegEnable(addr_to_dcache_div_set(x.bits.addr), x.valid)) 365 val write_div_addr_dup_reg = io.write_dup.map(x => RegEnable(addr_to_dcache_div(x.bits.addr), x.valid)) 366 367 // read data_banks and ecc_banks 368 // for single port SRAM, do not allow read and write in the same cycle 369 val rrhazard = false.B // io.readline.valid 370 (0 until LoadPipelineWidth).map(rport_index => { 371 div_addrs(rport_index) := addr_to_dcache_div(io.read(rport_index).bits.addr) 372 set_addrs(rport_index) := addr_to_dcache_div_set(io.read(rport_index).bits.addr) 373 bank_addrs(rport_index)(0) := addr_to_dcache_bank(io.read(rport_index).bits.addr) 374 bank_addrs(rport_index)(1) := bank_addrs(rport_index)(0) + 1.U 375 376 // use way_en to select a way after data read out 377 assert(!(RegNext(io.read(rport_index).fire && PopCount(io.read(rport_index).bits.way_en) > 1.U))) 378 way_en(rport_index) := io.read(rport_index).bits.way_en 379 }) 380 381 // read conflict 382 val rr_bank_conflict = Seq.tabulate(LoadPipelineWidth)(x => Seq.tabulate(LoadPipelineWidth)(y => 383 io.read(x).valid && io.read(y).valid && 384 div_addrs(x) === div_addrs(y) && 385 (io.read(x).bits.bankMask & io.read(y).bits.bankMask) =/= 0.U && 386 io.read(x).bits.way_en === io.read(y).bits.way_en && 387 set_addrs(x) =/= set_addrs(y) 388 )) 389 val rrl_bank_conflict = Wire(Vec(LoadPipelineWidth, Bool())) 390 val rrl_bank_conflict_intend = Wire(Vec(LoadPipelineWidth, Bool())) 391 (0 until LoadPipelineWidth).foreach { i => 392 val judge = if (ReduceReadlineConflict) io.read(i).valid && (io.readline.bits.rmask & io.read(i).bits.bankMask) =/= 0.U && line_div_addr === div_addrs(i) && line_set_addr =/= set_addrs(i) 393 else io.read(i).valid && line_div_addr === div_addrs(i) && line_set_addr =/= set_addrs(i) 394 rrl_bank_conflict(i) := judge && io.readline.valid 395 rrl_bank_conflict_intend(i) := judge && io.readline_intend 396 } 397 val wr_bank_conflict = Seq.tabulate(LoadPipelineWidth)(x => 398 io.read(x).valid && write_valid_reg && 399 div_addrs(x) === write_div_addr_dup_reg.head && 400 way_en(x) === write_wayen_dup_reg.head && 401 (write_bank_mask_reg(bank_addrs(x)(0)) || write_bank_mask_reg(bank_addrs(x)(1)) && io.is128Req(x)) 402 ) 403 val wrl_bank_conflict = io.readline.valid && write_valid_reg && line_div_addr === write_div_addr_dup_reg.head 404 // ready 405 io.readline.ready := !(wrl_bank_conflict) 406 io.read.zipWithIndex.map { case (x, i) => x.ready := !(wr_bank_conflict(i) || rrhazard) } 407 408 val perf_multi_read = PopCount(io.read.map(_.valid)) >= 2.U 409 val bank_conflict_fast = Wire(Vec(LoadPipelineWidth, Bool())) 410 (0 until LoadPipelineWidth).foreach(i => { 411 bank_conflict_fast(i) := wr_bank_conflict(i) || rrl_bank_conflict(i) || 412 (if (i == 0) 0.B else (0 until i).map(rr_bank_conflict(_)(i)).reduce(_ || _)) 413 io.bank_conflict_slow(i) := RegNext(bank_conflict_fast(i)) 414 io.disable_ld_fast_wakeup(i) := wr_bank_conflict(i) || rrl_bank_conflict_intend(i) || 415 (if (i == 0) 0.B else (0 until i).map(rr_bank_conflict(_)(i)).reduce(_ || _)) 416 }) 417 XSPerfAccumulate("data_array_multi_read", perf_multi_read) 418 (1 until LoadPipelineWidth).foreach(y => (0 until y).foreach(x => 419 XSPerfAccumulate(s"data_array_rr_bank_conflict_${x}_${y}", rr_bank_conflict(x)(y)) 420 )) 421 (0 until LoadPipelineWidth).foreach(i => { 422 XSPerfAccumulate(s"data_array_rrl_bank_conflict_${i}", rrl_bank_conflict(i)) 423 XSPerfAccumulate(s"data_array_rw_bank_conflict_${i}", wr_bank_conflict(i)) 424 XSPerfAccumulate(s"data_array_read_${i}", io.read(i).valid) 425 }) 426 XSPerfAccumulate("data_array_access_total", PopCount(io.read.map(_.valid))) 427 XSPerfAccumulate("data_array_read_line", io.readline.valid) 428 XSPerfAccumulate("data_array_write", io.write.valid) 429 430 val read_result = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, Vec(DCacheWays,new L1BankedDataReadResult())))) 431 val read_error_delayed_result = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, Vec(DCacheWays, Bool())))) 432 dontTouch(read_result) 433 dontTouch(read_error_delayed_result) 434 for (div_index <- 0 until DCacheSetDiv){ 435 for (bank_index <- 0 until DCacheBanks) { 436 for (way_index <- 0 until DCacheWays) { 437 // Set Addr & Read Way Mask 438 // 439 // Pipe 0 .... Pipe (n-1) 440 // + .... + 441 // | .... | 442 // +----+---------------+-----+ 443 // X X 444 // X +------+ Bank Addr Match 445 // +---------+----------+ 446 // | 447 // +--------+--------+ 448 // | Data Bank | 449 // +-----------------+ 450 val loadpipe_en = WireInit(VecInit(List.tabulate(LoadPipelineWidth)(i => { 451 io.read(i).valid && div_addrs(i) === div_index.U && (bank_addrs(i)(0) === bank_index.U || bank_addrs(i)(1) === bank_index.U && io.is128Req(i)) && way_en(i)(way_index) 452 }))) 453 val readline_en = Wire(Bool()) 454 if (ReduceReadlineConflict) { 455 readline_en := io.readline.valid && io.readline.bits.rmask(bank_index) && line_way_en(way_index) && div_index.U === line_div_addr 456 } else { 457 readline_en := io.readline.valid && line_way_en(way_index) && div_index.U === line_div_addr 458 } 459 val sram_set_addr = Mux(readline_en, 460 addr_to_dcache_div_set(io.readline.bits.addr), 461 PriorityMux(Seq.tabulate(LoadPipelineWidth)(i => loadpipe_en(i) -> set_addrs(i))) 462 ) 463 val read_en = loadpipe_en.asUInt.orR || readline_en 464 // read raw data 465 val data_bank = data_banks(div_index)(bank_index)(way_index) 466 data_bank.io.r.en := read_en 467 data_bank.io.r.addr := sram_set_addr 468 ecc_banks match { 469 case Some(banks) => 470 val ecc_bank = banks(div_index)(bank_index)(way_index) 471 ecc_bank.io.r.req.valid := read_en 472 ecc_bank.io.r.req.bits.apply(setIdx = sram_set_addr) 473 read_result(div_index)(bank_index)(way_index).ecc := ecc_bank.io.r.resp.data(0) 474 case None => 475 read_result(div_index)(bank_index)(way_index).ecc := 0.U 476 } 477 478 read_result(div_index)(bank_index)(way_index).raw_data := data_bank.io.r.data 479 480 // use ECC to check error 481 ecc_banks match { 482 case Some(_) => 483 val ecc_data = read_result(div_index)(bank_index)(way_index).asECCData() 484 val ecc_data_delayed = RegEnable(ecc_data, RegNext(read_en)) 485 read_result(div_index)(bank_index)(way_index).error_delayed := dcacheParameters.dataCode.decode(ecc_data_delayed).error 486 read_error_delayed_result(div_index)(bank_index)(way_index) := read_result(div_index)(bank_index)(way_index).error_delayed 487 case None => 488 read_result(div_index)(bank_index)(way_index).error_delayed := false.B 489 read_error_delayed_result(div_index)(bank_index)(way_index) := false.B 490 } 491 } 492 } 493 } 494 495 val data_read_oh = WireInit(VecInit(Seq.fill(DCacheSetDiv * DCacheBanks * DCacheWays)(0.U(1.W)))) 496 for(div_index <- 0 until DCacheSetDiv){ 497 for (bank_index <- 0 until DCacheBanks) { 498 for (way_index <- 0 until DCacheWays) { 499 data_read_oh(div_index * DCacheBanks * DCacheWays + bank_index * DCacheBanks + way_index) := data_banks(div_index)(bank_index)(way_index).io.r.en 500 } 501 } 502 } 503 XSPerfAccumulate("data_read_counter", PopCount(Cat(data_read_oh))) 504 505 // read result: expose banked read result 506 // TODO: clock gate 507 val read_result_delayed = RegNext(read_result) 508 (0 until LoadPipelineWidth).map(i => { 509 // io.read_resp(i) := read_result(RegNext(bank_addrs(i)))(RegNext(OHToUInt(way_en(i)))) 510 val rr_read_fire = RegNext(RegNext(io.read(i).fire)) 511 val rr_div_addr = RegNext(RegNext(div_addrs(i))) 512 val rr_bank_addr = RegNext(RegNext(bank_addrs(i))) 513 val rr_way_addr = RegNext(RegNext(OHToUInt(way_en(i)))) 514 (0 until VLEN/DCacheSRAMRowBits).map( j =>{ 515 io.read_resp_delayed(i)(j) := read_result_delayed(rr_div_addr)(rr_bank_addr(j))(rr_way_addr) 516 // error detection 517 // normal read ports 518 io.read_error_delayed(i)(j) := rr_read_fire && read_error_delayed_result(rr_div_addr)(rr_bank_addr(j))(rr_way_addr) && !RegNext(io.bank_conflict_slow(i)) 519 }) 520 }) 521 522 // readline port 523 (0 until DCacheBanks).map(i => { 524 io.readline_resp(i) := read_result(RegNext(line_div_addr))(i)(RegNext(OHToUInt(io.readline.bits.way_en))) 525 }) 526 io.readline_error_delayed := RegNext(RegNext(io.readline.fire)) && 527 VecInit((0 until DCacheBanks).map(i => io.readline_resp(i).error_delayed)).asUInt.orR 528 529 // write data_banks & ecc_banks 530 for (div_index <- 0 until DCacheSetDiv) { 531 for (bank_index <- 0 until DCacheBanks) { 532 for (way_index <- 0 until DCacheWays) { 533 // data write 534 val wen_reg = write_bank_mask_reg(bank_index) && 535 write_valid_dup_reg(bank_index) && 536 write_div_addr_dup_reg(bank_index) === div_index.U && 537 write_wayen_dup_reg(bank_index)(way_index) 538 val data_bank = data_banks(div_index)(bank_index)(way_index) 539 data_bank.io.w.en := wen_reg 540 541 data_bank.io.w.addr := write_set_addr_dup_reg(bank_index) 542 data_bank.io.w.data := write_data_reg(bank_index) 543 // ecc write 544 ecc_banks match { 545 case Some(banks) => 546 val ecc_bank = banks(div_index)(bank_index)(way_index) 547 ecc_bank.io.w.req.valid := wen_reg 548 ecc_bank.io.w.req.bits.apply( 549 setIdx = write_set_addr_dup_reg(bank_index), 550 data = RegNext(getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index))))), 551 waymask = 1.U 552 ) 553 when(ecc_bank.io.w.req.valid) { 554 XSDebug("write in ecc sram: bank %x set %x data %x waymask %x\n", 555 bank_index.U, 556 addr_to_dcache_div_set(io.write.bits.addr), 557 getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), 558 io.write.bits.way_en 559 ) 560 } 561 case None => None 562 } 563 } 564 } 565 } 566 567 require(nWays <= 32) 568 io.cacheOp.resp.bits := DontCare 569 val cacheOpShouldResp = WireInit(false.B) 570 val eccReadResult = Wire(Vec(DCacheBanks, UInt(eccBits.W))) 571 // DCacheDupNum is 16 572 // vec: the dupIdx for every bank and every group 573 val rdata_dup_vec = Seq(0,0,1,1,2,2,3,3) 574 val rdataEcc_dup_vec = Seq(4,4,5,5,6,6,7,7) 575 val wdata_dup_vec = Seq(8,8,9,9,10,10,11,11) 576 val wdataEcc_dup_vec = Seq(12,12,13,13,14,14,15,15) 577 val cacheOpDivAddr = set_to_dcache_div(io.cacheOp.req.bits.index) 578 val cacheOpSetAddr = set_to_dcache_div_set(io.cacheOp.req.bits.index) 579 val cacheOpWayNum = io.cacheOp.req.bits.wayNum(4, 0) 580 rdata_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 581 for (divIdx <- 0 until DCacheSetDiv){ 582 for (wayIdx <- 0 until DCacheWays) { 583 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isReadData(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 584 val data_bank = data_banks(divIdx)(bankIdx)(wayIdx) 585 data_bank.io.r.en := UIntToOH(io.cacheOp.req.bits.wayNum(4, 0))(wayIdx) && cacheOpDivAddr === divIdx.U 586 data_bank.io.r.addr := cacheOpSetAddr 587 cacheOpShouldResp := true.B 588 } 589 } 590 } 591 } 592 rdataEcc_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 593 for (divIdx <- 0 until DCacheSetDiv) { 594 for (wayIdx <- 0 until DCacheWays) { 595 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isReadDataECC(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 596 ecc_banks match { 597 case Some(banks) => 598 val ecc_bank = banks(divIdx)(bankIdx)(wayIdx) 599 ecc_bank.io.r.req.valid := true.B 600 ecc_bank.io.r.req.bits.setIdx := cacheOpSetAddr 601 cacheOpShouldResp := true.B 602 case None => 603 cacheOpShouldResp := true.B 604 } 605 } 606 } 607 } 608 } 609 wdata_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 610 for (divIdx <- 0 until DCacheSetDiv) { 611 for (wayIdx <- 0 until DCacheWays) { 612 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isWriteData(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 613 val data_bank = data_banks(divIdx)(bankIdx)(wayIdx) 614 data_bank.io.w.en := UIntToOH(io.cacheOp.req.bits.wayNum(4, 0))(wayIdx) && cacheOpDivAddr === divIdx.U 615 data_bank.io.w.addr := cacheOpSetAddr 616 data_bank.io.w.data := io.cacheOp.req.bits.write_data_vec(bankIdx) 617 cacheOpShouldResp := true.B 618 } 619 } 620 } 621 } 622 wdataEcc_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 623 for (divIdx <- 0 until DCacheSetDiv) { 624 for (wayIdx <- 0 until DCacheWays) { 625 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isWriteDataECC(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 626 ecc_banks match { 627 case Some(banks) => 628 val ecc_bank = banks(divIdx)(bankIdx)(wayIdx) 629 ecc_bank.io.w.req.valid := UIntToOH(io.cacheOp.req.bits.wayNum(4, 0))(wayIdx) && cacheOpDivAddr === divIdx.U 630 ecc_bank.io.w.req.bits.apply( 631 setIdx = cacheOpSetAddr, 632 data = io.cacheOp.req.bits.write_data_ecc, 633 waymask = 1.U 634 ) 635 cacheOpShouldResp := true.B 636 case None => 637 cacheOpShouldResp := true.B 638 } 639 } 640 } 641 } 642 } 643 io.cacheOp.resp.valid := RegNext(io.cacheOp.req.valid && cacheOpShouldResp) 644 for (bank_index <- 0 until DCacheBanks) { 645 io.cacheOp.resp.bits.read_data_vec(bank_index) := read_result(RegNext(cacheOpDivAddr))(bank_index)(RegNext(cacheOpWayNum)).raw_data 646 eccReadResult(bank_index) := read_result(RegNext(cacheOpDivAddr))(bank_index)(RegNext(cacheOpWayNum)).ecc 647 } 648 649 io.cacheOp.resp.bits.read_data_ecc := Mux(io.cacheOp.resp.valid, 650 eccReadResult(RegNext(io.cacheOp.req.bits.bank_num)), 651 0.U 652 ) 653 654 val tableName = "BankConflict" + p(XSCoreParamsKey).HartId.toString 655 val siteName = "BankedDataArray" + p(XSCoreParamsKey).HartId.toString 656 val bankConflictTable = ChiselDB.createTable(tableName, new BankConflictDB) 657 val bankConflictData = Wire(new BankConflictDB) 658 for (i <- 0 until LoadPipelineWidth) { 659 bankConflictData.set_index(i) := set_addrs(i) 660 bankConflictData.addr(i) := io.read(i).bits.addr 661 } 662 663 // FIXME: rr_bank_conflict(0)(1) no generalization 664 when(rr_bank_conflict(0)(1)) { 665 (0 until (VLEN/DCacheSRAMRowBits)).map(i => { 666 bankConflictData.bank_index(i) := bank_addrs(0)(i) 667 }) 668 bankConflictData.way_index := OHToUInt(way_en(0)) 669 bankConflictData.fake_rr_bank_conflict := set_addrs(0) === set_addrs(1) && div_addrs(0) === div_addrs(1) 670 }.otherwise { 671 (0 until (VLEN/DCacheSRAMRowBits)).map(i => { 672 bankConflictData.bank_index(i) := 0.U 673 }) 674 bankConflictData.way_index := 0.U 675 bankConflictData.fake_rr_bank_conflict := false.B 676 } 677 678 val isWriteBankConflictTable = Constantin.createRecord(s"isWriteBankConflictTable${p(XSCoreParamsKey).HartId}") 679 bankConflictTable.log( 680 data = bankConflictData, 681 en = isWriteBankConflictTable.orR && rr_bank_conflict(0)(1), 682 site = siteName, 683 clock = clock, 684 reset = reset 685 ) 686 687 (1 until LoadPipelineWidth).foreach(y => (0 until y).foreach(x => 688 XSPerfAccumulate(s"data_array_fake_rr_bank_conflict_${x}_${y}", rr_bank_conflict(x)(y) && set_addrs(x)===set_addrs(y) && div_addrs(x) === div_addrs(y)) 689 )) 690 691} 692 693// the smallest access unit is bank 694class BankedDataArray(implicit p: Parameters) extends AbstractBankedDataArray { 695 println(" DCacheType: BankedDataArray") 696 val ReduceReadlineConflict = false 697 698 io.write.ready := true.B 699 io.write_dup.foreach(_.ready := true.B) 700 701 val data_banks = List.fill(DCacheSetDiv)(List.tabulate(DCacheBanks)(i => Module(new DataSRAMBank(i)))) 702 val ecc_banks = DataEccParam.map { 703 case _ => 704 val ecc = List.fill(DCacheSetDiv)(List.fill(DCacheBanks)( 705 Module(new SRAMTemplate( 706 Bits(eccBits.W), 707 set = DCacheSets / DCacheSetDiv, 708 way = DCacheWays, 709 shouldReset = false, 710 holdRead = false, 711 singlePort = true 712 )) 713 )) 714 ecc 715 } 716 717 data_banks.map(_.map(_.dump())) 718 719 val way_en = Wire(Vec(LoadPipelineWidth, io.read(0).bits.way_en.cloneType)) 720 val set_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 721 val div_addrs = Wire(Vec(LoadPipelineWidth, UInt())) 722 val bank_addrs = Wire(Vec(LoadPipelineWidth, Vec(VLEN/DCacheSRAMRowBits, UInt()))) 723 val way_en_reg = Wire(Vec(LoadPipelineWidth, io.read(0).bits.way_en.cloneType)) 724 val set_addrs_reg = Wire(Vec(LoadPipelineWidth, UInt())) 725 726 val line_set_addr = addr_to_dcache_div_set(io.readline.bits.addr) 727 val line_div_addr = addr_to_dcache_div(io.readline.bits.addr) 728 val line_way_en = io.readline.bits.way_en 729 730 val write_bank_mask_reg = RegNext(io.write.bits.wmask) 731 val write_data_reg = RegEnable(io.write.bits.data, io.write.valid) 732 val write_valid_reg = RegNext(io.write.valid) 733 val write_valid_dup_reg = io.write_dup.map(x => RegNext(x.valid)) 734 val write_wayen_dup_reg = io.write_dup.map(x => RegNext(x.bits.way_en)) 735 val write_set_addr_dup_reg = io.write_dup.map(x => RegEnable(addr_to_dcache_div_set(x.bits.addr), x.valid)) 736 val write_div_addr_dup_reg = io.write_dup.map(x => RegEnable(addr_to_dcache_div(x.bits.addr), x.valid)) 737 738 // read data_banks and ecc_banks 739 // for single port SRAM, do not allow read and write in the same cycle 740 val rwhazard = RegNext(io.write.valid) 741 val rrhazard = false.B // io.readline.valid 742 (0 until LoadPipelineWidth).map(rport_index => { 743 div_addrs(rport_index) := addr_to_dcache_div(io.read(rport_index).bits.addr) 744 bank_addrs(rport_index)(0) := addr_to_dcache_bank(io.read(rport_index).bits.addr) 745 bank_addrs(rport_index)(1) := Mux(io.is128Req(rport_index), bank_addrs(rport_index)(0) + 1.U, DCacheBanks.asUInt) 746 set_addrs(rport_index) := addr_to_dcache_div_set(io.read(rport_index).bits.addr) 747 set_addrs_reg(rport_index) := RegEnable(addr_to_dcache_div_set(io.read(rport_index).bits.addr), io.read(rport_index).valid) 748 749 // use way_en to select a way after data read out 750 assert(!(RegNext(io.read(rport_index).fire && PopCount(io.read(rport_index).bits.way_en) > 1.U))) 751 way_en(rport_index) := io.read(rport_index).bits.way_en 752 way_en_reg(rport_index) := RegEnable(io.read(rport_index).bits.way_en, io.read(rport_index).valid) 753 }) 754 755 // read each bank, get bank result 756 val rr_bank_conflict = Seq.tabulate(LoadPipelineWidth)(x => Seq.tabulate(LoadPipelineWidth)(y => 757 io.read(x).valid && io.read(y).valid && 758 div_addrs(x) === div_addrs(y) && 759 (io.read(x).bits.bankMask & io.read(y).bits.bankMask) =/= 0.U 760 )) 761 val rrl_bank_conflict = Wire(Vec(LoadPipelineWidth, Bool())) 762 val rrl_bank_conflict_intend = Wire(Vec(LoadPipelineWidth, Bool())) 763 (0 until LoadPipelineWidth).foreach { i => 764 val judge = if (ReduceReadlineConflict) io.read(i).valid && (io.readline.bits.rmask & io.read(i).bits.bankMask) =/= 0.U && div_addrs(i) === line_div_addr 765 else io.read(i).valid && div_addrs(i)===line_div_addr 766 rrl_bank_conflict(i) := judge && io.readline.valid 767 rrl_bank_conflict_intend(i) := judge && io.readline_intend 768 } 769 val wr_bank_conflict = Seq.tabulate(LoadPipelineWidth)(x => 770 io.read(x).valid && 771 write_valid_reg && 772 div_addrs(x) === write_div_addr_dup_reg.head && 773 (write_bank_mask_reg(bank_addrs(x)(0)) || write_bank_mask_reg(bank_addrs(x)(1)) && io.is128Req(x)) 774 ) 775 val wrl_bank_conflict = io.readline.valid && write_valid_reg && line_div_addr === write_div_addr_dup_reg.head 776 // ready 777 io.readline.ready := !(wrl_bank_conflict) 778 io.read.zipWithIndex.map{case(x, i) => x.ready := !(wr_bank_conflict(i) || rrhazard)} 779 780 val perf_multi_read = PopCount(io.read.map(_.valid)) >= 2.U 781 (0 until LoadPipelineWidth).foreach(i => { 782 // remove fake rr_bank_conflict situation in s2 783 val real_other_bank_conflict_reg = RegNext(wr_bank_conflict(i) || rrl_bank_conflict(i)) 784 val real_rr_bank_conflict_reg = (if (i == 0) 0.B else (0 until i).map{ j => 785 RegNext(rr_bank_conflict(j)(i)) && 786 (way_en_reg(j) =/= way_en_reg(i) || set_addrs_reg(j) =/= set_addrs_reg(i)) 787 }.reduce(_ || _)) 788 io.bank_conflict_slow(i) := real_other_bank_conflict_reg || real_rr_bank_conflict_reg 789 790 // get result in s1 791 io.disable_ld_fast_wakeup(i) := wr_bank_conflict(i) || rrl_bank_conflict_intend(i) || 792 (if (i == 0) 0.B else (0 until i).map(rr_bank_conflict(_)(i)).reduce(_ || _)) 793 }) 794 XSPerfAccumulate("data_array_multi_read", perf_multi_read) 795 (1 until LoadPipelineWidth).foreach(y => (0 until y).foreach(x => 796 XSPerfAccumulate(s"data_array_rr_bank_conflict_${x}_${y}", rr_bank_conflict(x)(y)) 797 )) 798 (0 until LoadPipelineWidth).foreach(i => { 799 XSPerfAccumulate(s"data_array_rrl_bank_conflict_${i}", rrl_bank_conflict(i)) 800 XSPerfAccumulate(s"data_array_rw_bank_conflict_${i}", wr_bank_conflict(i)) 801 XSPerfAccumulate(s"data_array_read_${i}", io.read(i).valid) 802 }) 803 XSPerfAccumulate("data_array_access_total", PopCount(io.read.map(_.valid))) 804 XSPerfAccumulate("data_array_read_line", io.readline.valid) 805 XSPerfAccumulate("data_array_write", io.write.valid) 806 807 val bank_result = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, new L1BankedDataReadResult()))) 808 val ecc_result = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, Vec(DCacheWays, UInt(eccBits.W))))) 809 val read_bank_error_delayed = Wire(Vec(DCacheSetDiv, Vec(DCacheBanks, Bool()))) 810 dontTouch(bank_result) 811 dontTouch(read_bank_error_delayed) 812 for (div_index <- 0 until DCacheSetDiv) { 813 for (bank_index <- 0 until DCacheBanks) { 814 // Set Addr & Read Way Mask 815 // 816 // Pipe 0 .... Pipe (n-1) 817 // + .... + 818 // | .... | 819 // +----+---------------+-----+ 820 // X X 821 // X +------+ Bank Addr Match 822 // +---------+----------+ 823 // | 824 // +--------+--------+ 825 // | Data Bank | 826 // +-----------------+ 827 val bank_addr_matchs = WireInit(VecInit(List.tabulate(LoadPipelineWidth)(i => { 828 io.read(i).valid && div_addrs(i) === div_index.U && (bank_addrs(i)(0) === bank_index.U || bank_addrs(i)(1) === bank_index.U && io.is128Req(i)) 829 }))) 830 val readline_match = Wire(Bool()) 831 if (ReduceReadlineConflict) { 832 readline_match := io.readline.valid && io.readline.bits.rmask(bank_index) && line_div_addr === div_index.U 833 } else { 834 readline_match := io.readline.valid && line_div_addr === div_index.U 835 } 836 val bank_way_en = Mux(readline_match, 837 io.readline.bits.way_en, 838 PriorityMux(Seq.tabulate(LoadPipelineWidth)(i => bank_addr_matchs(i) -> way_en(i))) 839 ) 840 // it is too long of bank_way_en's caculation, so bank_way_en_reg can not be caculated by RegNext(bank_way_en) 841 val bank_way_en_reg = Mux(RegNext(readline_match), 842 RegNext(io.readline.bits.way_en), 843 PriorityMux(Seq.tabulate(LoadPipelineWidth)(i => RegNext(bank_addr_matchs(i)) -> RegNext(way_en(i)))) 844 ) 845 val bank_set_addr = Mux(readline_match, 846 line_set_addr, 847 PriorityMux(Seq.tabulate(LoadPipelineWidth)(i => bank_addr_matchs(i) -> set_addrs(i))) 848 ) 849 850 val read_enable = bank_addr_matchs.asUInt.orR || readline_match 851 852 // read raw data 853 val data_bank = data_banks(div_index)(bank_index) 854 data_bank.io.r.en := read_enable 855 data_bank.io.r.way_en := bank_way_en 856 data_bank.io.r.addr := bank_set_addr 857 bank_result(div_index)(bank_index).raw_data := data_bank.io.r.data 858 859 // read ECC 860 ecc_banks match { 861 case Some(banks) => 862 val ecc_bank = banks(div_index)(bank_index) 863 ecc_bank.io.r.req.valid := read_enable 864 ecc_bank.io.r.req.bits.apply(setIdx = bank_set_addr) 865 ecc_result(div_index)(bank_index) := ecc_bank.io.r.resp.data 866 bank_result(div_index)(bank_index).ecc := Mux1H(bank_way_en_reg, ecc_bank.io.r.resp.data) 867 case None => 868 ecc_result(div_index)(bank_index) := DontCare 869 bank_result(div_index)(bank_index).ecc := DontCare 870 } 871 872 // use ECC to check error 873 ecc_banks match { 874 case Some(_) => 875 val ecc_data = bank_result(div_index)(bank_index).asECCData() 876 val ecc_data_delayed = RegEnable(ecc_data, RegNext(read_enable)) 877 bank_result(div_index)(bank_index).error_delayed := dcacheParameters.dataCode.decode(ecc_data_delayed).error 878 read_bank_error_delayed(div_index)(bank_index) := bank_result(div_index)(bank_index).error_delayed 879 case None => 880 bank_result(div_index)(bank_index).error_delayed := false.B 881 read_bank_error_delayed(div_index)(bank_index) := false.B 882 } 883 } 884 } 885 886 val data_read_oh = WireInit(VecInit(Seq.fill(DCacheSetDiv)(0.U(XLEN.W)))) 887 for (div_index <- 0 until DCacheSetDiv){ 888 val temp = WireInit(VecInit(Seq.fill(DCacheBanks)(0.U(XLEN.W)))) 889 for (bank_index <- 0 until DCacheBanks) { 890 temp(bank_index) := PopCount(Fill(DCacheWays, data_banks(div_index)(bank_index).io.r.en.asUInt)) 891 } 892 data_read_oh(div_index) := temp.reduce(_ + _) 893 } 894 XSPerfAccumulate("data_read_counter", data_read_oh.foldLeft(0.U)(_ + _)) 895 896 val bank_result_delayed = RegNext(bank_result) 897 (0 until LoadPipelineWidth).map(i => { 898 val r_read_fire = RegNext(io.read(i).fire) 899 val rr_read_fire = RegNext(r_read_fire) 900 val rr_div_addr = RegEnable(RegEnable(div_addrs(i), io.read(i).fire), r_read_fire) 901 val rr_bank_addr = RegEnable(RegEnable(bank_addrs(i), io.read(i).fire), r_read_fire) 902 val rr_way_addr = RegEnable(RegEnable(OHToUInt(way_en(i)), io.read(i).fire), r_read_fire) 903 (0 until VLEN/DCacheSRAMRowBits).map( j =>{ 904 io.read_resp_delayed(i)(j) := bank_result_delayed(rr_div_addr)(rr_bank_addr(j)) 905 // error detection 906 io.read_error_delayed(i)(j) := rr_read_fire && read_bank_error_delayed(rr_div_addr)(rr_bank_addr(j)) && !RegNext(io.bank_conflict_slow(i)) 907 }) 908 }) 909 910 // read result: expose banked read result 911 io.readline_resp := bank_result(RegEnable(line_div_addr, io.readline.valid)) 912 io.readline_error_delayed := RegNext(RegNext(io.readline.fire)) && 913 VecInit((0 until DCacheBanks).map(i => io.readline_resp(i).error_delayed)).asUInt.orR 914 915 // write data_banks & ecc_banks 916 for (div_index <- 0 until DCacheSetDiv) { 917 for (bank_index <- 0 until DCacheBanks) { 918 // data write 919 val wen_reg = write_bank_mask_reg(bank_index) && 920 write_valid_dup_reg(bank_index) && 921 write_div_addr_dup_reg(bank_index) === div_index.U 922 val data_bank = data_banks(div_index)(bank_index) 923 data_bank.io.w.en := wen_reg 924 data_bank.io.w.way_en := write_wayen_dup_reg(bank_index) 925 data_bank.io.w.addr := write_set_addr_dup_reg(bank_index) 926 data_bank.io.w.data := write_data_reg(bank_index) 927 928 // ecc write 929 ecc_banks match { 930 case Some(banks) => 931 val ecc_bank = banks(div_index)(bank_index) 932 ecc_bank.io.w.req.valid := wen_reg 933 ecc_bank.io.w.req.bits.apply( 934 setIdx = write_set_addr_dup_reg(bank_index), 935 data = RegEnable(getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), wen_reg), 936 waymask = write_wayen_dup_reg(bank_index) 937 ) 938 when(ecc_bank.io.w.req.valid) { 939 XSDebug("write in ecc sram: bank %x set %x data %x waymask %x\n", 940 bank_index.U, 941 addr_to_dcache_div_set(io.write.bits.addr), 942 getECCFromEncWord(cacheParams.dataCode.encode((io.write.bits.data(bank_index)))), 943 io.write.bits.way_en 944 ) 945 } 946 case None => None 947 } 948 } 949 } 950 951 // deal with customized cache op 952 require(nWays <= 32) 953 io.cacheOp.resp.bits := DontCare 954 val cacheOpShouldResp = WireInit(false.B) 955 val eccReadResult = Wire(Vec(DCacheBanks, UInt(eccBits.W))) 956 // DCacheDupNum is 16 957 // vec: the dupIdx for every bank and every group 958 val rdata_dup_vec = Seq(0, 0, 1, 1, 2, 2, 3, 3) 959 val rdataEcc_dup_vec = Seq(4, 4, 5, 5, 6, 6, 7, 7) 960 val wdata_dup_vec = Seq(8, 8, 9, 9, 10, 10, 11, 11) 961 val wdataEcc_dup_vec = Seq(12, 12, 13, 13, 14, 14, 15, 15) 962 val cacheOpDivAddr = set_to_dcache_div(io.cacheOp.req.bits.index) 963 val cacheOpSetAddr = set_to_dcache_div_set(io.cacheOp.req.bits.index) 964 val cacheOpWayMask = UIntToOH(io.cacheOp.req.bits.wayNum(4, 0)) 965 rdata_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 966 for (divIdx <- 0 until DCacheSetDiv) { 967 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isReadData(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 968 val data_bank = data_banks(divIdx)(bankIdx) 969 data_bank.io.r.en := true.B 970 data_bank.io.r.way_en := cacheOpWayMask 971 data_bank.io.r.addr := cacheOpSetAddr 972 cacheOpShouldResp := true.B 973 } 974 } 975 } 976 rdataEcc_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 977 for (divIdx <- 0 until DCacheSetDiv) { 978 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isReadDataECC(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 979 ecc_banks match { 980 case Some(banks) => 981 val ecc_bank = banks(divIdx)(bankIdx) 982 ecc_bank.io.r.req.valid := true.B 983 ecc_bank.io.r.req.bits.setIdx := cacheOpSetAddr 984 cacheOpShouldResp := true.B 985 case None => 986 cacheOpShouldResp := true.B 987 } 988 } 989 } 990 } 991 wdata_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 992 for (divIdx <- 0 until DCacheSetDiv) { 993 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isWriteData(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 994 val data_bank = data_banks(divIdx)(bankIdx) 995 data_bank.io.w.en := cacheOpDivAddr === divIdx.U 996 data_bank.io.w.way_en := cacheOpWayMask 997 data_bank.io.w.addr := cacheOpSetAddr 998 data_bank.io.w.data := io.cacheOp.req.bits.write_data_vec(bankIdx) 999 cacheOpShouldResp := true.B 1000 } 1001 } 1002 } 1003 wdataEcc_dup_vec.zipWithIndex.map{ case(dupIdx, bankIdx) => 1004 for (divIdx <- 0 until DCacheSetDiv) { 1005 when(io.cacheOp_req_dup(dupIdx).valid && CacheInstrucion.isWriteDataECC(io.cacheOp_req_bits_opCode_dup(dupIdx))) { 1006 ecc_banks match { 1007 case Some(banks) => 1008 val ecc_bank = banks(divIdx)(bankIdx) 1009 ecc_bank.io.w.req.valid := cacheOpDivAddr === divIdx.U 1010 ecc_bank.io.w.req.bits.apply( 1011 setIdx = cacheOpSetAddr, 1012 data = io.cacheOp.req.bits.write_data_ecc, 1013 waymask = cacheOpWayMask 1014 ) 1015 cacheOpShouldResp := true.B 1016 case None => 1017 cacheOpShouldResp := true.B 1018 } 1019 } 1020 } 1021 } 1022 1023 io.cacheOp.resp.valid := RegNext(io.cacheOp.req.valid && cacheOpShouldResp) 1024 for (bank_index <- 0 until DCacheBanks) { 1025 io.cacheOp.resp.bits.read_data_vec(bank_index) := bank_result(RegNext(cacheOpDivAddr))(bank_index).raw_data 1026 eccReadResult(bank_index) := Mux1H(RegNext(cacheOpWayMask), ecc_result(RegNext(cacheOpDivAddr))(bank_index)) 1027 } 1028 1029 io.cacheOp.resp.bits.read_data_ecc := Mux(io.cacheOp.resp.valid, 1030 eccReadResult(RegNext(io.cacheOp.req.bits.bank_num)), 1031 0.U 1032 ) 1033 1034 val tableName = "BankConflict" + p(XSCoreParamsKey).HartId.toString 1035 val siteName = "BankedDataArray" + p(XSCoreParamsKey).HartId.toString 1036 val bankConflictTable = ChiselDB.createTable(tableName, new BankConflictDB) 1037 val bankConflictData = Wire(new BankConflictDB) 1038 for (i <- 0 until LoadPipelineWidth) { 1039 bankConflictData.set_index(i) := set_addrs(i) 1040 bankConflictData.addr(i) := io.read(i).bits.addr 1041 } 1042 1043 // FIXME: rr_bank_conflict(0)(1) no generalization 1044 when(rr_bank_conflict(0)(1)) { 1045 (0 until (VLEN/DCacheSRAMRowBits)).map(i => { 1046 bankConflictData.bank_index(i) := bank_addrs(0)(i) 1047 }) 1048 bankConflictData.way_index := OHToUInt(way_en(0)) 1049 bankConflictData.fake_rr_bank_conflict := set_addrs(0) === set_addrs(1) && div_addrs(0) === div_addrs(1) 1050 }.otherwise { 1051 (0 until (VLEN/DCacheSRAMRowBits)).map(i => { 1052 bankConflictData.bank_index(i) := 0.U 1053 }) 1054 bankConflictData.way_index := 0.U 1055 bankConflictData.fake_rr_bank_conflict := false.B 1056 } 1057 1058 val isWriteBankConflictTable = Constantin.createRecord(s"isWriteBankConflictTable${p(XSCoreParamsKey).HartId}") 1059 bankConflictTable.log( 1060 data = bankConflictData, 1061 en = isWriteBankConflictTable.orR && rr_bank_conflict(0)(1), 1062 site = siteName, 1063 clock = clock, 1064 reset = reset 1065 ) 1066 1067 (1 until LoadPipelineWidth).foreach(y => (0 until y).foreach(x => 1068 XSPerfAccumulate(s"data_array_fake_rr_bank_conflict_${x}_${y}", rr_bank_conflict(x)(y) && set_addrs(x) === set_addrs(y) && div_addrs(x) === div_addrs(y)) 1069 )) 1070 1071} 1072