Lines Matching full:qp
12 void erdma_qp_llp_close(struct erdma_qp *qp) in erdma_qp_llp_close() argument
16 down_write(&qp->state_lock); in erdma_qp_llp_close()
18 switch (qp->attrs.iwarp.state) { in erdma_qp_llp_close()
24 erdma_modify_qp_state_iwarp(qp, ¶ms, ERDMA_QPA_IWARP_STATE); in erdma_qp_llp_close()
27 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_IDLE; in erdma_qp_llp_close()
33 if (qp->cep) { in erdma_qp_llp_close()
34 erdma_cep_put(qp->cep); in erdma_qp_llp_close()
35 qp->cep = NULL; in erdma_qp_llp_close()
38 up_write(&qp->state_lock); in erdma_qp_llp_close()
43 struct erdma_qp *qp = find_qp_by_qpn(to_edev(ibdev), id); in erdma_get_ibqp() local
45 if (qp) in erdma_get_ibqp()
46 return &qp->ibqp; in erdma_get_ibqp()
52 erdma_modify_qp_state_to_rts(struct erdma_qp *qp, in erdma_modify_qp_state_to_rts() argument
57 struct erdma_dev *dev = qp->dev; in erdma_modify_qp_state_to_rts()
60 struct erdma_cep *cep = qp->cep; in erdma_modify_qp_state_to_rts()
70 params->cc = qp->attrs.cc; in erdma_modify_qp_state_to_rts()
80 tp = tcp_sk(qp->cep->sock->sk); in erdma_modify_qp_state_to_rts()
87 FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp)); in erdma_modify_qp_state_to_rts()
107 qp->attrs.irq_size = params->irq_size; in erdma_modify_qp_state_to_rts()
110 qp->attrs.orq_size = params->orq_size; in erdma_modify_qp_state_to_rts()
113 qp->attrs.cc = params->cc; in erdma_modify_qp_state_to_rts()
115 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_RTS; in erdma_modify_qp_state_to_rts()
121 erdma_modify_qp_state_to_stop(struct erdma_qp *qp, in erdma_modify_qp_state_to_stop() argument
125 struct erdma_dev *dev = qp->dev; in erdma_modify_qp_state_to_stop()
133 FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp)); in erdma_modify_qp_state_to_stop()
140 qp->attrs.iwarp.state = params->state; in erdma_modify_qp_state_to_stop()
145 int erdma_modify_qp_state_iwarp(struct erdma_qp *qp, in erdma_modify_qp_state_iwarp() argument
158 switch (qp->attrs.iwarp.state) { in erdma_modify_qp_state_iwarp()
162 ret = erdma_modify_qp_state_to_rts(qp, params, mask); in erdma_modify_qp_state_iwarp()
164 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_ERROR; in erdma_modify_qp_state_iwarp()
166 if (qp->cep) { in erdma_modify_qp_state_iwarp()
167 erdma_cep_put(qp->cep); in erdma_modify_qp_state_iwarp()
168 qp->cep = NULL; in erdma_modify_qp_state_iwarp()
170 ret = erdma_modify_qp_state_to_stop(qp, params, mask); in erdma_modify_qp_state_iwarp()
179 ret = erdma_modify_qp_state_to_stop(qp, params, mask); in erdma_modify_qp_state_iwarp()
185 erdma_qp_cm_drop(qp); in erdma_modify_qp_state_iwarp()
190 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_ERROR; in erdma_modify_qp_state_iwarp()
194 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_IDLE; in erdma_modify_qp_state_iwarp()
196 ret = erdma_modify_qp_state_to_stop(qp, params, mask); in erdma_modify_qp_state_iwarp()
197 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_ERROR; in erdma_modify_qp_state_iwarp()
206 if (need_reflush && !ret && rdma_is_kernel_res(&qp->ibqp.res)) { in erdma_modify_qp_state_iwarp()
207 qp->flags |= ERDMA_QP_IN_FLUSHING; in erdma_modify_qp_state_iwarp()
208 mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork, in erdma_modify_qp_state_iwarp()
215 static int modify_qp_cmd_rocev2(struct erdma_qp *qp, in modify_qp_cmd_rocev2() argument
226 req.cfg0 = FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp)); in modify_qp_cmd_rocev2()
250 return erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, in modify_qp_cmd_rocev2()
254 static void erdma_reset_qp(struct erdma_qp *qp) in erdma_reset_qp() argument
256 qp->kern_qp.sq_pi = 0; in erdma_reset_qp()
257 qp->kern_qp.sq_ci = 0; in erdma_reset_qp()
258 qp->kern_qp.rq_pi = 0; in erdma_reset_qp()
259 qp->kern_qp.rq_ci = 0; in erdma_reset_qp()
260 memset(qp->kern_qp.swr_tbl, 0, qp->attrs.sq_size * sizeof(u64)); in erdma_reset_qp()
261 memset(qp->kern_qp.rwr_tbl, 0, qp->attrs.rq_size * sizeof(u64)); in erdma_reset_qp()
262 memset(qp->kern_qp.sq_buf, 0, qp->attrs.sq_size << SQEBB_SHIFT); in erdma_reset_qp()
263 memset(qp->kern_qp.rq_buf, 0, qp->attrs.rq_size << RQE_SHIFT); in erdma_reset_qp()
264 erdma_remove_cqes_of_qp(&qp->scq->ibcq, QP_ID(qp)); in erdma_reset_qp()
265 if (qp->rcq != qp->scq) in erdma_reset_qp()
266 erdma_remove_cqes_of_qp(&qp->rcq->ibcq, QP_ID(qp)); in erdma_reset_qp()
269 int erdma_modify_qp_state_rocev2(struct erdma_qp *qp, in erdma_modify_qp_state_rocev2() argument
273 struct erdma_dev *dev = to_edev(qp->ibqp.device); in erdma_modify_qp_state_rocev2()
276 ret = modify_qp_cmd_rocev2(qp, params, attr_mask); in erdma_modify_qp_state_rocev2()
281 qp->attrs.rocev2.state = params->state; in erdma_modify_qp_state_rocev2()
284 qp->attrs.rocev2.qkey = params->qkey; in erdma_modify_qp_state_rocev2()
287 qp->attrs.rocev2.dst_qpn = params->dst_qpn; in erdma_modify_qp_state_rocev2()
290 memcpy(&qp->attrs.rocev2.av, ¶ms->av, in erdma_modify_qp_state_rocev2()
293 if (rdma_is_kernel_res(&qp->ibqp.res) && in erdma_modify_qp_state_rocev2()
295 erdma_reset_qp(qp); in erdma_modify_qp_state_rocev2()
297 if (rdma_is_kernel_res(&qp->ibqp.res) && in erdma_modify_qp_state_rocev2()
299 qp->flags |= ERDMA_QP_IN_FLUSHING; in erdma_modify_qp_state_rocev2()
300 mod_delayed_work(dev->reflush_wq, &qp->reflush_dwork, in erdma_modify_qp_state_rocev2()
309 struct erdma_qp *qp = container_of(ref, struct erdma_qp, ref); in erdma_qp_safe_free() local
311 complete(&qp->safe_free); in erdma_qp_safe_free()
314 void erdma_qp_put(struct erdma_qp *qp) in erdma_qp_put() argument
316 WARN_ON(kref_read(&qp->ref) < 1); in erdma_qp_put()
317 kref_put(&qp->ref, erdma_qp_safe_free); in erdma_qp_put()
320 void erdma_qp_get(struct erdma_qp *qp) in erdma_qp_get() argument
322 kref_get(&qp->ref); in erdma_qp_get()
325 static int fill_inline_data(struct erdma_qp *qp, in fill_inline_data() argument
335 data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size, in fill_inline_data()
359 data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, in fill_inline_data()
360 qp->attrs.sq_size, SQEBB_SHIFT); in fill_inline_data()
372 static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr, in fill_sgl() argument
379 if (send_wr->num_sge > qp->dev->attrs.max_send_sge) in fill_sgl()
388 sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, in fill_sgl()
389 qp->attrs.sq_size, SQEBB_SHIFT); in fill_sgl()
403 static void init_send_sqe_rc(struct erdma_qp *qp, struct erdma_send_sqe_rc *sqe, in init_send_sqe_rc() argument
419 static void init_send_sqe_ud(struct erdma_qp *qp, struct erdma_send_sqe_ud *sqe, in init_send_sqe_ud() argument
437 sqe->qkey = cpu_to_le32(qp->attrs.rocev2.qkey); in init_send_sqe_ud()
442 static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi, in erdma_push_one_sqe() argument
446 u32 idx = *pi & (qp->attrs.sq_size - 1); in erdma_push_one_sqe()
462 if (qp->ibqp.qp_type != IB_QPT_RC && send_wr->opcode != IB_WR_SEND && in erdma_push_one_sqe()
466 entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size, in erdma_push_one_sqe()
472 qp->kern_qp.swr_tbl[idx] = send_wr->wr_id; in erdma_push_one_sqe()
476 ((flags & IB_SEND_SIGNALED) || qp->kern_qp.sig_all) ? 1 : 0); in erdma_push_one_sqe()
483 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp)); in erdma_push_one_sqe()
527 sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1, in erdma_push_one_sqe()
528 qp->attrs.sq_size, SQEBB_SHIFT); in erdma_push_one_sqe()
539 if (qp->ibqp.qp_type == IB_QPT_RC) { in erdma_push_one_sqe()
541 init_send_sqe_rc(qp, rc_send_sqe, send_wr, &hw_op); in erdma_push_one_sqe()
546 init_send_sqe_ud(qp, ud_send_sqe, send_wr, &hw_op); in erdma_push_one_sqe()
572 memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1, in erdma_push_one_sqe()
573 qp->attrs.sq_size, SQEBB_SHIFT), in erdma_push_one_sqe()
608 sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1, in erdma_push_one_sqe()
609 qp->attrs.sq_size, SQEBB_SHIFT); in erdma_push_one_sqe()
625 ret = fill_inline_data(qp, send_wr, idx, sgl_offset, in erdma_push_one_sqe()
632 ret = fill_sgl(qp, send_wr, idx, sgl_offset, length_field); in erdma_push_one_sqe()
651 static void kick_sq_db(struct erdma_qp *qp, u16 pi) in kick_sq_db() argument
653 u64 db_data = FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp)) | in kick_sq_db()
656 *(u64 *)qp->kern_qp.sq_dbrec = db_data; in kick_sq_db()
657 writeq(db_data, qp->kern_qp.hw_sq_db); in kick_sq_db()
663 struct erdma_qp *qp = to_eqp(ibqp); in erdma_post_send() local
672 spin_lock_irqsave(&qp->lock, flags); in erdma_post_send()
673 sq_pi = qp->kern_qp.sq_pi; in erdma_post_send()
676 if ((u16)(sq_pi - qp->kern_qp.sq_ci) >= qp->attrs.sq_size) { in erdma_post_send()
682 ret = erdma_push_one_sqe(qp, &sq_pi, wr); in erdma_post_send()
687 qp->kern_qp.sq_pi = sq_pi; in erdma_post_send()
688 kick_sq_db(qp, sq_pi); in erdma_post_send()
692 spin_unlock_irqrestore(&qp->lock, flags); in erdma_post_send()
694 if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING)) in erdma_post_send()
695 mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork, in erdma_post_send()
701 static int erdma_post_recv_one(struct erdma_qp *qp, in erdma_post_recv_one() argument
705 get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi, in erdma_post_recv_one()
706 qp->attrs.rq_size, RQE_SHIFT); in erdma_post_recv_one()
708 rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1); in erdma_post_recv_one()
709 rqe->qpn = cpu_to_le32(QP_ID(qp)); in erdma_post_recv_one()
721 *(u64 *)qp->kern_qp.rq_dbrec = *(u64 *)rqe; in erdma_post_recv_one()
722 writeq(*(u64 *)rqe, qp->kern_qp.hw_rq_db); in erdma_post_recv_one()
724 qp->kern_qp.rwr_tbl[qp->kern_qp.rq_pi & (qp->attrs.rq_size - 1)] = in erdma_post_recv_one()
726 qp->kern_qp.rq_pi++; in erdma_post_recv_one()
735 struct erdma_qp *qp = to_eqp(ibqp); in erdma_post_recv() local
739 spin_lock_irqsave(&qp->lock, flags); in erdma_post_recv()
742 ret = erdma_post_recv_one(qp, wr); in erdma_post_recv()
750 spin_unlock_irqrestore(&qp->lock, flags); in erdma_post_recv()
752 if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING)) in erdma_post_recv()
753 mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork, in erdma_post_recv()