xref: /XiangShan/src/main/scala/xiangshan/mem/pipeline/LoadUnit.scala (revision c7658a75968a011d92bf164a1a55872e64f06d44)
1package xiangshan.mem
2
3import chisel3._
4import chisel3.util._
5import utils._
6import xiangshan._
7import xiangshan.cache._
8// import xiangshan.cache.{DCacheWordIO, TlbRequestIO, TlbCmd, MemoryOpConstants, TlbReq, DCacheLoadReq, DCacheWordResp}
9import xiangshan.backend.LSUOpType
10import xiangshan.backend.fu.fpu.boxF32ToF64
11
12class LoadToLsroqIO extends XSBundle {
13  val loadIn = ValidIO(new LsPipelineBundle)
14  val ldout = Flipped(DecoupledIO(new ExuOutput))
15  val forward = new LoadForwardQueryIO
16}
17
18// Load Pipeline Stage 0
19// Generate addr, use addr to query DCache and DTLB
20class LoadUnit_S0 extends XSModule {
21  val io = IO(new Bundle() {
22    val in = Flipped(Decoupled(new ExuInput))
23    val out = Decoupled(new LsPipelineBundle)
24    val redirect = Flipped(ValidIO(new Redirect))
25    val dtlbReq = Valid(new TlbReq)
26    val dtlbResp = Flipped(Valid(new TlbResp))
27    val tlbFeedback = ValidIO(new TlbFeedback)
28    val dcacheReq = DecoupledIO(new DCacheLoadReq)
29  })
30
31  val s0_uop = io.in.bits.uop
32  val s0_vaddr = io.in.bits.src1 + s0_uop.ctrl.imm
33  val s0_paddr = io.dtlbResp.bits.paddr
34  val s0_tlb_miss = io.dtlbResp.bits.miss
35  val s0_mask = genWmask(s0_vaddr, s0_uop.ctrl.fuOpType(1,0))
36
37  // query DTLB
38  io.dtlbReq.valid := io.out.valid
39  io.dtlbReq.bits.vaddr := s0_vaddr
40  io.dtlbReq.bits.cmd := TlbCmd.read
41  io.dtlbReq.bits.roqIdx := s0_uop.roqIdx
42  io.dtlbReq.bits.debug.pc := s0_uop.cf.pc
43
44  // feedback tlb result to RS
45  // Note: can be moved to s1
46  io.tlbFeedback.valid := io.out.valid
47  io.tlbFeedback.bits.hit := !s0_tlb_miss
48  io.tlbFeedback.bits.roqIdx := s0_uop.roqIdx
49
50  // query DCache
51  io.dcacheReq.valid := io.in.valid && !s0_uop.roqIdx.needFlush(io.redirect)
52  io.dcacheReq.bits.cmd  := MemoryOpConstants.M_XRD
53  io.dcacheReq.bits.addr := s0_vaddr
54  io.dcacheReq.bits.mask := s0_mask
55  io.dcacheReq.bits.data := DontCare
56
57  // TODO: update cache meta
58  io.dcacheReq.bits.meta.id       := DontCare
59  io.dcacheReq.bits.meta.vaddr    := s0_vaddr
60  io.dcacheReq.bits.meta.paddr    := DontCare
61  io.dcacheReq.bits.meta.uop      := s0_uop
62  io.dcacheReq.bits.meta.mmio     := false.B
63  io.dcacheReq.bits.meta.tlb_miss := false.B
64  io.dcacheReq.bits.meta.mask     := s0_mask
65  io.dcacheReq.bits.meta.replay   := false.B
66
67  val addrAligned = LookupTree(s0_uop.ctrl.fuOpType(1, 0), List(
68    "b00".U   -> true.B,                   //b
69    "b01".U   -> (s0_vaddr(0)    === 0.U), //h
70    "b10".U   -> (s0_vaddr(1, 0) === 0.U), //w
71    "b11".U   -> (s0_vaddr(2, 0) === 0.U)  //d
72  ))
73
74  io.out.valid := io.dcacheReq.fire() // dcache may not accept load request
75  io.out.bits := DontCare
76  io.out.bits.vaddr := s0_vaddr
77  io.out.bits.paddr := s0_paddr
78  io.out.bits.tlbMiss := io.dtlbResp.bits.miss
79  io.out.bits.mask := s0_mask
80  io.out.bits.uop := s0_uop
81  io.out.bits.uop.cf.exceptionVec(loadAddrMisaligned) := !addrAligned
82  io.out.bits.uop.cf.exceptionVec(loadPageFault) := io.dtlbResp.bits.excp.pf.ld
83
84  io.in.ready := io.out.fire()
85
86  XSDebug(io.dcacheReq.fire(), "[DCACHE LOAD REQ] pc %x vaddr %x paddr will be %x\n",
87    s0_uop.cf.pc, s0_vaddr, s0_paddr
88  )
89}
90
91
92// Load Pipeline Stage 1
93// TLB resp (send paddr to dcache)
94class LoadUnit_S1 extends XSModule {
95  val io = IO(new Bundle() {
96    val in = Flipped(Decoupled(new LsPipelineBundle))
97    val out = Decoupled(new LsPipelineBundle)
98    val redirect = Flipped(ValidIO(new Redirect))
99    val s1_paddr = Output(UInt(PAddrBits.W))
100    val sbuffer = new LoadForwardQueryIO
101    val lsroq = new LoadForwardQueryIO
102  })
103
104  val s1_uop = io.in.bits.uop
105  val s1_paddr = io.in.bits.paddr
106  val s1_tlb_miss = io.in.bits.tlbMiss
107  val s1_mmio = !s1_tlb_miss && AddressSpace.isMMIO(s1_paddr)
108  val s1_mask = io.in.bits.mask
109
110  io.out.bits := io.in.bits // forwardXX field will be updated in s1
111  io.s1_paddr :=  s1_paddr
112
113  // load forward query datapath
114  io.sbuffer.valid := io.in.valid
115  io.sbuffer.paddr := s1_paddr
116  io.sbuffer.uop := s1_uop
117  io.sbuffer.sqIdx := s1_uop.sqIdx
118  io.sbuffer.mask := s1_mask
119  io.sbuffer.pc := s1_uop.cf.pc // FIXME: remove it
120
121  io.lsroq.valid := io.in.valid
122  io.lsroq.paddr := s1_paddr
123  io.lsroq.uop := s1_uop
124  io.lsroq.sqIdx := s1_uop.sqIdx
125  io.lsroq.mask := s1_mask
126  io.lsroq.pc := s1_uop.cf.pc // FIXME: remove it
127
128  io.out.bits.forwardMask := io.sbuffer.forwardMask
129  io.out.bits.forwardData := io.sbuffer.forwardData
130  // generate XLEN/8 Muxs
131  for (i <- 0 until XLEN / 8) {
132    when(io.lsroq.forwardMask(i)) {
133      io.out.bits.forwardMask(i) := true.B
134      io.out.bits.forwardData(i) := io.lsroq.forwardData(i)
135    }
136  }
137
138  XSDebug(io.out.fire(), "[FWD LOAD RESP] pc %x fwd %x(%b) + %x(%b)\n",
139    s1_uop.cf.pc,
140    io.lsroq.forwardData.asUInt, io.lsroq.forwardMask.asUInt,
141    io.sbuffer.forwardData.asUInt, io.sbuffer.forwardMask.asUInt
142  )
143
144  io.out.valid := io.in.valid && !s1_tlb_miss &&  !s1_uop.roqIdx.needFlush(io.redirect)
145  io.out.bits.paddr := s1_paddr
146  io.out.bits.mmio := s1_mmio
147  io.out.bits.tlbMiss := s1_tlb_miss
148
149  io.in.ready := io.out.ready || !io.in.valid
150
151}
152
153
154// Load Pipeline Stage 2
155// DCache resp
156class LoadUnit_S2 extends XSModule {
157  val io = IO(new Bundle() {
158    val in = Flipped(Decoupled(new LsPipelineBundle))
159    val out = Decoupled(new LsPipelineBundle)
160    val redirect = Flipped(ValidIO(new Redirect))
161    val dcacheResp = Flipped(DecoupledIO(new DCacheWordResp))
162  })
163
164  val s2_uop = io.in.bits.uop
165  val s2_mask = io.in.bits.mask
166  val s2_paddr = io.in.bits.paddr
167  val s2_cache_miss = io.dcacheResp.bits.miss
168  val s2_cache_nack = io.dcacheResp.bits.nack
169
170
171  io.dcacheResp.ready := true.B
172  assert(!(io.in.valid && !io.dcacheResp.valid), "DCache response got lost")
173
174  val forwardMask = io.in.bits.forwardMask
175  val forwardData = io.in.bits.forwardData
176  val fullForward = (~forwardMask.asUInt & s2_mask) === 0.U
177
178  // data merge
179  val rdata = VecInit((0 until XLEN / 8).map(j =>
180    Mux(forwardMask(j), forwardData(j), io.dcacheResp.bits.data(8*(j+1)-1, 8*j)))).asUInt
181  val rdataSel = LookupTree(s2_paddr(2, 0), List(
182    "b000".U -> rdata(63, 0),
183    "b001".U -> rdata(63, 8),
184    "b010".U -> rdata(63, 16),
185    "b011".U -> rdata(63, 24),
186    "b100".U -> rdata(63, 32),
187    "b101".U -> rdata(63, 40),
188    "b110".U -> rdata(63, 48),
189    "b111".U -> rdata(63, 56)
190  ))
191  val rdataPartialLoad = LookupTree(s2_uop.ctrl.fuOpType, List(
192      LSUOpType.lb   -> SignExt(rdataSel(7, 0) , XLEN),
193      LSUOpType.lh   -> SignExt(rdataSel(15, 0), XLEN),
194      LSUOpType.lw   -> SignExt(rdataSel(31, 0), XLEN),
195      LSUOpType.ld   -> SignExt(rdataSel(63, 0), XLEN),
196      LSUOpType.lbu  -> ZeroExt(rdataSel(7, 0) , XLEN),
197      LSUOpType.lhu  -> ZeroExt(rdataSel(15, 0), XLEN),
198      LSUOpType.lwu  -> ZeroExt(rdataSel(31, 0), XLEN),
199      LSUOpType.flw  -> boxF32ToF64(rdataSel(31, 0))
200  ))
201
202  // TODO: ECC check
203
204  io.out.valid := io.in.valid // && !s2_uop.needFlush(io.redirect) will cause comb. loop
205  // Inst will be canceled in store queue / lsroq,
206  // so we do not need to care about flush in load / store unit's out.valid
207  io.out.bits := io.in.bits
208  io.out.bits.data := rdataPartialLoad
209  io.out.bits.miss := (s2_cache_miss || s2_cache_nack) && !fullForward
210  io.out.bits.mmio := io.in.bits.mmio
211
212  io.in.ready := io.out.ready || !io.in.valid
213
214  XSDebug(io.out.fire(), "[DCACHE LOAD RESP] pc %x rdata %x <- D$ %x + fwd %x(%b)\n",
215    s2_uop.cf.pc, rdataPartialLoad, io.dcacheResp.bits.data,
216    io.in.bits.forwardData.asUInt, io.in.bits.forwardMask.asUInt
217  )
218
219}
220
221
222class LoadUnit extends XSModule {
223  val io = IO(new Bundle() {
224    val ldin = Flipped(Decoupled(new ExuInput))
225    val ldout = Decoupled(new ExuOutput)
226    val redirect = Flipped(ValidIO(new Redirect))
227    val tlbFeedback = ValidIO(new TlbFeedback)
228    val dcache = new DCacheLoadIO
229    val dtlb = new TlbRequestIO()
230    val sbuffer = new LoadForwardQueryIO
231    val lsroq = new LoadToLsroqIO
232  })
233
234  val load_s0 = Module(new LoadUnit_S0)
235  val load_s1 = Module(new LoadUnit_S1)
236  val load_s2 = Module(new LoadUnit_S2)
237
238  load_s0.io.in <> io.ldin
239  load_s0.io.redirect <> io.redirect
240  load_s0.io.dtlbReq <> io.dtlb.req
241  load_s0.io.dtlbResp <> io.dtlb.resp
242  load_s0.io.dcacheReq <> io.dcache.req
243  load_s0.io.tlbFeedback <> io.tlbFeedback
244
245  PipelineConnect(load_s0.io.out, load_s1.io.in, load_s1.io.out.fire() || load_s1.io.out.bits.uop.roqIdx.needFlush(io.redirect), false.B)
246
247  io.dcache.s1_paddr := load_s1.io.out.bits.paddr
248  load_s1.io.redirect <> io.redirect
249  io.dcache.s1_kill := DontCare // FIXME
250  io.sbuffer <> load_s1.io.sbuffer
251  io.lsroq.forward <> load_s1.io.lsroq
252
253  PipelineConnect(load_s1.io.out, load_s2.io.in, load_s2.io.out.fire() || load_s1.io.out.bits.tlbMiss, false.B)
254
255  load_s2.io.redirect <> io.redirect
256  load_s2.io.dcacheResp <> io.dcache.resp
257
258  XSDebug(load_s0.io.out.valid,
259    p"S0: pc ${Hexadecimal(load_s0.io.out.bits.uop.cf.pc)}, lId ${Hexadecimal(load_s0.io.out.bits.uop.lqIdx.asUInt)}, " +
260    p"vaddr ${Hexadecimal(load_s0.io.out.bits.vaddr)}, mask ${Hexadecimal(load_s0.io.out.bits.mask)}\n")
261  XSDebug(load_s1.io.out.valid,
262    p"S1: pc ${Hexadecimal(load_s1.io.out.bits.uop.cf.pc)}, lId ${Hexadecimal(load_s1.io.out.bits.uop.lqIdx.asUInt)}, tlb_miss ${io.dtlb.resp.bits.miss}, " +
263    p"paddr ${Hexadecimal(load_s1.io.out.bits.paddr)}, mmio ${load_s1.io.out.bits.mmio}\n")
264
265  // writeback to LSROQ
266  // Current dcache use MSHR
267  io.lsroq.loadIn.valid := load_s2.io.out.valid
268  io.lsroq.loadIn.bits := load_s2.io.out.bits
269
270  val hitLoadOut = Wire(Valid(new ExuOutput))
271  hitLoadOut.valid := load_s2.io.out.valid && !load_s2.io.out.bits.miss
272  hitLoadOut.bits.uop := load_s2.io.out.bits.uop
273  hitLoadOut.bits.data := load_s2.io.out.bits.data
274  hitLoadOut.bits.redirectValid := false.B
275  hitLoadOut.bits.redirect := DontCare
276  hitLoadOut.bits.brUpdate := DontCare
277  hitLoadOut.bits.debug.isMMIO := load_s2.io.out.bits.mmio
278  hitLoadOut.bits.fflags := DontCare
279
280  // TODO: arbiter
281  // if hit, writeback result to CDB
282  // val ldout = Vec(2, Decoupled(new ExuOutput))
283  // when io.loadIn(i).fire() && !io.io.loadIn(i).miss, commit load to cdb
284  // val cdbArb = Module(new Arbiter(new ExuOutput, 2))
285  // io.ldout <> cdbArb.io.out
286  // hitLoadOut <> cdbArb.io.in(0)
287  // io.lsroq.ldout <> cdbArb.io.in(1) // missLoadOut
288  load_s2.io.out.ready := true.B
289  io.lsroq.ldout.ready := !hitLoadOut.valid
290  io.ldout.bits := Mux(hitLoadOut.valid, hitLoadOut.bits, io.lsroq.ldout.bits)
291  io.ldout.valid := hitLoadOut.valid || io.lsroq.ldout.valid
292
293  when(io.ldout.fire()){
294    XSDebug("ldout %x iw %x fw %x\n", io.ldout.bits.uop.cf.pc, io.ldout.bits.uop.ctrl.rfWen, io.ldout.bits.uop.ctrl.fpWen)
295  }
296}
297