Lines Matching +full:post +full:-
2 * Copyright (c) 2015-2016 QLogic Corporation
14 * - Redistributions of source code must retain the above
18 * - Redistributions in binary form must reproduce the above
32 #include <linux/dma-mapping.h>
50 #include <rdma/qedr-abi.h>
55 info->gsi_cons = (info->gsi_cons + 1) % info->max_wr; in qedr_inc_sw_gsi_cons()
61 dev->gsi_qp_created = 1; in qedr_store_gsi_qp_cq()
62 dev->gsi_sqcq = get_qedr_cq(attrs->send_cq); in qedr_store_gsi_qp_cq()
63 dev->gsi_rqcq = get_qedr_cq(attrs->recv_cq); in qedr_store_gsi_qp_cq()
64 dev->gsi_qp = qp; in qedr_store_gsi_qp_cq()
75 struct qedr_cq *cq = dev->gsi_sqcq; in qedr_ll2_complete_tx_packet()
76 struct qedr_qp *qp = dev->gsi_qp; in qedr_ll2_complete_tx_packet()
81 dev->gsi_sqcq, dev->gsi_rqcq, qp->sq.gsi_cons, in qedr_ll2_complete_tx_packet()
82 cq->ibcq.comp_handler ? "Yes" : "No"); in qedr_ll2_complete_tx_packet()
84 dma_free_coherent(&dev->pdev->dev, pkt->header.len, pkt->header.vaddr, in qedr_ll2_complete_tx_packet()
85 pkt->header.baddr); in qedr_ll2_complete_tx_packet()
88 spin_lock_irqsave(&qp->q_lock, flags); in qedr_ll2_complete_tx_packet()
89 qedr_inc_sw_gsi_cons(&qp->sq); in qedr_ll2_complete_tx_packet()
90 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_ll2_complete_tx_packet()
92 if (cq->ibcq.comp_handler) in qedr_ll2_complete_tx_packet()
93 (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); in qedr_ll2_complete_tx_packet()
100 struct qedr_cq *cq = dev->gsi_rqcq; in qedr_ll2_complete_rx_packet()
101 struct qedr_qp *qp = dev->gsi_qp; in qedr_ll2_complete_rx_packet()
104 spin_lock_irqsave(&qp->q_lock, flags); in qedr_ll2_complete_rx_packet()
106 qp->rqe_wr_id[qp->rq.gsi_cons].rc = data->u.data_length_error ? in qedr_ll2_complete_rx_packet()
107 -EINVAL : 0; in qedr_ll2_complete_rx_packet()
108 qp->rqe_wr_id[qp->rq.gsi_cons].vlan = data->vlan; in qedr_ll2_complete_rx_packet()
110 qp->rqe_wr_id[qp->rq.gsi_cons].sg_list[0].length = in qedr_ll2_complete_rx_packet()
111 data->length.data_length; in qedr_ll2_complete_rx_packet()
112 *((u32 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[0]) = in qedr_ll2_complete_rx_packet()
113 ntohl(data->opaque_data_0); in qedr_ll2_complete_rx_packet()
114 *((u16 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[4]) = in qedr_ll2_complete_rx_packet()
115 ntohs((u16)data->opaque_data_1); in qedr_ll2_complete_rx_packet()
117 qedr_inc_sw_gsi_cons(&qp->rq); in qedr_ll2_complete_rx_packet()
119 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_ll2_complete_rx_packet()
121 if (cq->ibcq.comp_handler) in qedr_ll2_complete_rx_packet()
122 (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); in qedr_ll2_complete_rx_packet()
139 cq = get_qedr_cq(attrs->send_cq); in qedr_destroy_gsi_cq()
140 iparams.icid = cq->icid; in qedr_destroy_gsi_cq()
141 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); in qedr_destroy_gsi_cq()
142 dev->ops->common->chain_free(dev->cdev, &cq->pbl); in qedr_destroy_gsi_cq()
144 cq = get_qedr_cq(attrs->recv_cq); in qedr_destroy_gsi_cq()
146 if (iparams.icid != cq->icid) { in qedr_destroy_gsi_cq()
147 iparams.icid = cq->icid; in qedr_destroy_gsi_cq()
148 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); in qedr_destroy_gsi_cq()
149 dev->ops->common->chain_free(dev->cdev, &cq->pbl); in qedr_destroy_gsi_cq()
156 if (attrs->cap.max_recv_sge > QEDR_GSI_MAX_RECV_SGE) { in qedr_check_gsi_qp_attrs()
159 attrs->cap.max_recv_sge, QEDR_GSI_MAX_RECV_SGE); in qedr_check_gsi_qp_attrs()
160 return -EINVAL; in qedr_check_gsi_qp_attrs()
163 if (attrs->cap.max_recv_wr > QEDR_GSI_MAX_RECV_WR) { in qedr_check_gsi_qp_attrs()
166 attrs->cap.max_recv_wr, QEDR_GSI_MAX_RECV_WR); in qedr_check_gsi_qp_attrs()
167 return -EINVAL; in qedr_check_gsi_qp_attrs()
170 if (attrs->cap.max_send_wr > QEDR_GSI_MAX_SEND_WR) { in qedr_check_gsi_qp_attrs()
173 attrs->cap.max_send_wr, QEDR_GSI_MAX_SEND_WR); in qedr_check_gsi_qp_attrs()
174 return -EINVAL; in qedr_check_gsi_qp_attrs()
190 roce_flavor = (pkt->roce_mode == ROCE_V1) ? in qedr_ll2_post_tx()
193 if (pkt->roce_mode == ROCE_V2_IPV4) in qedr_ll2_post_tx()
196 ll2_tx_pkt.num_of_bds = 1 /* hdr */ + pkt->n_seg; in qedr_ll2_post_tx()
198 ll2_tx_pkt.tx_dest = pkt->tx_dest; in qedr_ll2_post_tx()
200 ll2_tx_pkt.first_frag = pkt->header.baddr; in qedr_ll2_post_tx()
201 ll2_tx_pkt.first_frag_len = pkt->header.len; in qedr_ll2_post_tx()
205 rc = dev->ops->ll2_prepare_tx_packet(dev->rdma_ctx, in qedr_ll2_post_tx()
206 dev->gsi_ll2_handle, in qedr_ll2_post_tx()
209 /* TX failed while posting header - release resources */ in qedr_ll2_post_tx()
210 dma_free_coherent(&dev->pdev->dev, pkt->header.len, in qedr_ll2_post_tx()
211 pkt->header.vaddr, pkt->header.baddr); in qedr_ll2_post_tx()
219 for (i = 0; i < pkt->n_seg; i++) { in qedr_ll2_post_tx()
220 rc = dev->ops->ll2_set_fragment_of_tx_packet( in qedr_ll2_post_tx()
221 dev->rdma_ctx, in qedr_ll2_post_tx()
222 dev->gsi_ll2_handle, in qedr_ll2_post_tx()
223 pkt->payload[i].baddr, in qedr_ll2_post_tx()
224 pkt->payload[i].len); in qedr_ll2_post_tx()
243 if (dev->gsi_ll2_handle == QED_LL2_UNUSED_HANDLE) in qedr_ll2_stop()
247 rc = dev->ops->ll2_set_mac_filter(dev->cdev, in qedr_ll2_stop()
248 dev->gsi_ll2_mac_address, NULL); in qedr_ll2_stop()
250 rc = dev->ops->ll2_terminate_connection(dev->rdma_ctx, in qedr_ll2_stop()
251 dev->gsi_ll2_handle); in qedr_ll2_stop()
255 dev->ops->ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); in qedr_ll2_stop()
257 dev->gsi_ll2_handle = QED_LL2_UNUSED_HANDLE; in qedr_ll2_stop()
278 data.input.mtu = dev->ndev->mtu; in qedr_ll2_start()
279 data.input.rx_num_desc = attrs->cap.max_recv_wr; in qedr_ll2_start()
282 data.input.tx_num_desc = attrs->cap.max_send_wr; in qedr_ll2_start()
288 data.p_connection_handle = &dev->gsi_ll2_handle; in qedr_ll2_start()
291 rc = dev->ops->ll2_acquire_connection(dev->rdma_ctx, &data); in qedr_ll2_start()
299 rc = dev->ops->ll2_establish_connection(dev->rdma_ctx, in qedr_ll2_start()
300 dev->gsi_ll2_handle); in qedr_ll2_start()
308 rc = dev->ops->ll2_set_mac_filter(dev->cdev, NULL, dev->ndev->dev_addr); in qedr_ll2_start()
315 dev->ops->ll2_terminate_connection(dev->rdma_ctx, dev->gsi_ll2_handle); in qedr_ll2_start()
317 dev->ops->ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); in qedr_ll2_start()
338 qp->ibqp.qp_num = 1; in qedr_create_gsi_qp()
339 qp->rq.max_wr = attrs->cap.max_recv_wr; in qedr_create_gsi_qp()
340 qp->sq.max_wr = attrs->cap.max_send_wr; in qedr_create_gsi_qp()
342 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), in qedr_create_gsi_qp()
344 if (!qp->rqe_wr_id) in qedr_create_gsi_qp()
346 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), in qedr_create_gsi_qp()
348 if (!qp->wqe_wr_id) in qedr_create_gsi_qp()
352 ether_addr_copy(dev->gsi_ll2_mac_address, dev->ndev->dev_addr); in qedr_create_gsi_qp()
356 dev->gsi_rqcq->cq_type = QEDR_CQ_TYPE_GSI; in qedr_create_gsi_qp()
363 kfree(qp->rqe_wr_id); in qedr_create_gsi_qp()
369 return -ENOMEM; in qedr_create_gsi_qp()
386 struct rdma_ah_attr *ah_attr = &get_qedr_ah(ud_wr(swr)->ah)->attr; in qedr_gsi_build_header()
388 const struct ib_gid_attr *sgid_attr = grh->sgid_attr; in qedr_gsi_build_header()
406 for (i = 0; i < swr->num_sge; ++i) in qedr_gsi_build_header()
407 send_size += swr->sg_list[i].length; in qedr_gsi_build_header()
409 has_udp = (sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP); in qedr_gsi_build_header()
414 } else if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) { in qedr_gsi_build_header()
430 DP_ERR(dev, "gsi post send: failed to init header\n"); in qedr_gsi_build_header()
435 ether_addr_copy(udh->eth.dmac_h, ah_attr->roce.dmac); in qedr_gsi_build_header()
436 ether_addr_copy(udh->eth.smac_h, dev->ndev->dev_addr); in qedr_gsi_build_header()
438 udh->eth.type = htons(ETH_P_8021Q); in qedr_gsi_build_header()
439 udh->vlan.tag = htons(vlan_id); in qedr_gsi_build_header()
440 udh->vlan.type = htons(ether_type); in qedr_gsi_build_header()
442 udh->eth.type = htons(ether_type); in qedr_gsi_build_header()
446 udh->bth.solicited_event = !!(swr->send_flags & IB_SEND_SOLICITED); in qedr_gsi_build_header()
447 udh->bth.pkey = QEDR_ROCE_PKEY_DEFAULT; in qedr_gsi_build_header()
448 udh->bth.destination_qpn = htonl(ud_wr(swr)->remote_qpn); in qedr_gsi_build_header()
449 udh->bth.psn = htonl((qp->sq_psn++) & ((1 << 24) - 1)); in qedr_gsi_build_header()
450 udh->bth.opcode = IB_OPCODE_UD_SEND_ONLY; in qedr_gsi_build_header()
453 udh->deth.qkey = htonl(0x80010000); in qedr_gsi_build_header()
454 udh->deth.source_qpn = htonl(QEDR_GSI_QPN); in qedr_gsi_build_header()
458 udh->grh.traffic_class = grh->traffic_class; in qedr_gsi_build_header()
459 udh->grh.flow_label = grh->flow_label; in qedr_gsi_build_header()
460 udh->grh.hop_limit = grh->hop_limit; in qedr_gsi_build_header()
461 udh->grh.destination_gid = grh->dgid; in qedr_gsi_build_header()
462 memcpy(&udh->grh.source_gid.raw, sgid_attr->gid.raw, in qedr_gsi_build_header()
463 sizeof(udh->grh.source_gid.raw)); in qedr_gsi_build_header()
468 udh->ip4.protocol = IPPROTO_UDP; in qedr_gsi_build_header()
469 udh->ip4.tos = htonl(grh->flow_label); in qedr_gsi_build_header()
470 udh->ip4.frag_off = htons(IP_DF); in qedr_gsi_build_header()
471 udh->ip4.ttl = grh->hop_limit; in qedr_gsi_build_header()
473 ipv4_addr = qedr_get_ipv4_from_gid(sgid_attr->gid.raw); in qedr_gsi_build_header()
474 udh->ip4.saddr = ipv4_addr; in qedr_gsi_build_header()
475 ipv4_addr = qedr_get_ipv4_from_gid(grh->dgid.raw); in qedr_gsi_build_header()
476 udh->ip4.daddr = ipv4_addr; in qedr_gsi_build_header()
482 udh->udp.sport = htons(QEDR_ROCE_V2_UDP_SPORT); in qedr_gsi_build_header()
483 udh->udp.dport = htons(ROCE_V2_UDP_DPORT); in qedr_gsi_build_header()
484 udh->udp.csum = 0; in qedr_gsi_build_header()
497 struct pci_dev *pdev = dev->pdev; in qedr_gsi_build_packet()
512 return -ENOMEM; in qedr_gsi_build_packet()
514 packet->header.vaddr = dma_alloc_coherent(&pdev->dev, header_size, in qedr_gsi_build_packet()
515 &packet->header.baddr, in qedr_gsi_build_packet()
517 if (!packet->header.vaddr) { in qedr_gsi_build_packet()
519 return -ENOMEM; in qedr_gsi_build_packet()
523 packet->tx_dest = QED_LL2_TX_DEST_LB; in qedr_gsi_build_packet()
525 packet->tx_dest = QED_LL2_TX_DEST_NW; in qedr_gsi_build_packet()
527 packet->roce_mode = roce_mode; in qedr_gsi_build_packet()
528 memcpy(packet->header.vaddr, ud_header_buffer, header_size); in qedr_gsi_build_packet()
529 packet->header.len = header_size; in qedr_gsi_build_packet()
530 packet->n_seg = swr->num_sge; in qedr_gsi_build_packet()
531 for (i = 0; i < packet->n_seg; i++) { in qedr_gsi_build_packet()
532 packet->payload[i].baddr = swr->sg_list[i].addr; in qedr_gsi_build_packet()
533 packet->payload[i].len = swr->sg_list[i].length; in qedr_gsi_build_packet()
546 struct qedr_dev *dev = qp->dev; in qedr_gsi_post_send()
550 if (qp->state != QED_ROCE_QP_STATE_RTS) { in qedr_gsi_post_send()
553 "gsi post recv: failed to post rx buffer. state is %d and not QED_ROCE_QP_STATE_RTS\n", in qedr_gsi_post_send()
554 qp->state); in qedr_gsi_post_send()
555 return -EINVAL; in qedr_gsi_post_send()
558 if (wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE) { in qedr_gsi_post_send()
559 DP_ERR(dev, "gsi post send: num_sge is too large (%d>%d)\n", in qedr_gsi_post_send()
560 wr->num_sge, RDMA_MAX_SGE_PER_SQ_WQE); in qedr_gsi_post_send()
561 rc = -EINVAL; in qedr_gsi_post_send()
565 if (wr->opcode != IB_WR_SEND) { in qedr_gsi_post_send()
567 "gsi post send: failed due to unsupported opcode %d\n", in qedr_gsi_post_send()
568 wr->opcode); in qedr_gsi_post_send()
569 rc = -EINVAL; in qedr_gsi_post_send()
573 spin_lock_irqsave(&qp->q_lock, flags); in qedr_gsi_post_send()
577 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_send()
584 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; in qedr_gsi_post_send()
585 qedr_inc_sw_prod(&qp->sq); in qedr_gsi_post_send()
586 DP_DEBUG(qp->dev, QEDR_MSG_GSI, in qedr_gsi_post_send()
587 "gsi post send: opcode=%d, wr_id=%llx\n", wr->opcode, in qedr_gsi_post_send()
588 wr->wr_id); in qedr_gsi_post_send()
590 DP_ERR(dev, "gsi post send: failed to transmit (rc=%d)\n", rc); in qedr_gsi_post_send()
591 rc = -EAGAIN; in qedr_gsi_post_send()
595 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_send()
597 if (wr->next) { in qedr_gsi_post_send()
599 "gsi post send: failed second WR. Only one WR may be passed at a time\n"); in qedr_gsi_post_send()
600 *bad_wr = wr->next; in qedr_gsi_post_send()
601 rc = -EINVAL; in qedr_gsi_post_send()
614 struct qedr_dev *dev = get_qedr_dev(ibqp->device); in qedr_gsi_post_recv()
619 if ((qp->state != QED_ROCE_QP_STATE_RTR) && in qedr_gsi_post_recv()
620 (qp->state != QED_ROCE_QP_STATE_RTS)) { in qedr_gsi_post_recv()
623 "gsi post recv: failed to post rx buffer. state is %d and not QED_ROCE_QP_STATE_RTR/S\n", in qedr_gsi_post_recv()
624 qp->state); in qedr_gsi_post_recv()
625 return -EINVAL; in qedr_gsi_post_recv()
628 spin_lock_irqsave(&qp->q_lock, flags); in qedr_gsi_post_recv()
631 if (wr->num_sge > QEDR_GSI_MAX_RECV_SGE) { in qedr_gsi_post_recv()
633 "gsi post recv: failed to post rx buffer. too many sges %d>%d\n", in qedr_gsi_post_recv()
634 wr->num_sge, QEDR_GSI_MAX_RECV_SGE); in qedr_gsi_post_recv()
638 rc = dev->ops->ll2_post_rx_buffer(dev->rdma_ctx, in qedr_gsi_post_recv()
639 dev->gsi_ll2_handle, in qedr_gsi_post_recv()
640 wr->sg_list[0].addr, in qedr_gsi_post_recv()
641 wr->sg_list[0].length, in qedr_gsi_post_recv()
646 "gsi post recv: failed to post rx buffer (rc=%d)\n", in qedr_gsi_post_recv()
651 memset(&qp->rqe_wr_id[qp->rq.prod], 0, in qedr_gsi_post_recv()
652 sizeof(qp->rqe_wr_id[qp->rq.prod])); in qedr_gsi_post_recv()
653 qp->rqe_wr_id[qp->rq.prod].sg_list[0] = wr->sg_list[0]; in qedr_gsi_post_recv()
654 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; in qedr_gsi_post_recv()
656 qedr_inc_sw_prod(&qp->rq); in qedr_gsi_post_recv()
658 wr = wr->next; in qedr_gsi_post_recv()
661 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_recv()
665 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_recv()
667 return -ENOMEM; in qedr_gsi_post_recv()
672 struct qedr_dev *dev = get_qedr_dev(ibcq->device); in qedr_gsi_poll_cq()
674 struct qedr_qp *qp = dev->gsi_qp; in qedr_gsi_poll_cq()
679 spin_lock_irqsave(&cq->cq_lock, flags); in qedr_gsi_poll_cq()
681 while (i < num_entries && qp->rq.cons != qp->rq.gsi_cons) { in qedr_gsi_poll_cq()
684 wc[i].qp = &qp->ibqp; in qedr_gsi_poll_cq()
685 wc[i].wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; in qedr_gsi_poll_cq()
688 wc[i].status = (qp->rqe_wr_id[qp->rq.cons].rc) ? in qedr_gsi_poll_cq()
690 /* 0 - currently only one recv sg is supported */ in qedr_gsi_poll_cq()
691 wc[i].byte_len = qp->rqe_wr_id[qp->rq.cons].sg_list[0].length; in qedr_gsi_poll_cq()
693 ether_addr_copy(wc[i].smac, qp->rqe_wr_id[qp->rq.cons].smac); in qedr_gsi_poll_cq()
696 vlan_id = qp->rqe_wr_id[qp->rq.cons].vlan & in qedr_gsi_poll_cq()
701 wc[i].sl = (qp->rqe_wr_id[qp->rq.cons].vlan & in qedr_gsi_poll_cq()
705 qedr_inc_sw_cons(&qp->rq); in qedr_gsi_poll_cq()
709 while (i < num_entries && qp->sq.cons != qp->sq.gsi_cons) { in qedr_gsi_poll_cq()
712 wc[i].qp = &qp->ibqp; in qedr_gsi_poll_cq()
713 wc[i].wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; in qedr_gsi_poll_cq()
717 qedr_inc_sw_cons(&qp->sq); in qedr_gsi_poll_cq()
721 spin_unlock_irqrestore(&cq->cq_lock, flags); in qedr_gsi_poll_cq()
724 …"gsi poll_cq: requested entries=%d, actual=%d, qp->rq.cons=%d, qp->rq.gsi_cons=%x, qp->sq.cons=%d,… in qedr_gsi_poll_cq()
725 num_entries, i, qp->rq.cons, qp->rq.gsi_cons, qp->sq.cons, in qedr_gsi_poll_cq()
726 qp->sq.gsi_cons, qp->ibqp.qp_num); in qedr_gsi_poll_cq()