2  * Copyright (c) 2015-2016  QLogic Corporation
14  *      - Redistributions of source code must retain the above
18  *      - Redistributions in binary form must reproduce the above
32 #include <linux/dma-mapping.h>
52 #include <rdma/qedr-abi.h>
70 	size_t min_len = min_t(size_t, len, udata->outlen);  in qedr_ib_copy_to_udata()
78 		return -EINVAL;  in qedr_query_pkey()
89 	memset(sgid->raw, 0, sizeof(sgid->raw));  in qedr_iw_query_gid()
90 	ether_addr_copy(sgid->raw, dev->ndev->dev_addr);  in qedr_iw_query_gid()
93 		 sgid->global.interface_id, sgid->global.subnet_prefix);  in qedr_iw_query_gid()
100 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);  in qedr_query_srq()
101 	struct qedr_device_attr *qattr = &dev->attr;  in qedr_query_srq()
104 	srq_attr->srq_limit = srq->srq_limit;  in qedr_query_srq()
105 	srq_attr->max_wr = qattr->max_srq_wr;  in qedr_query_srq()
106 	srq_attr->max_sge = qattr->max_sge;  in qedr_query_srq()
115 	struct qedr_device_attr *qattr = &dev->attr;  in qedr_query_device()
117 	if (!dev->rdma_ctx) {  in qedr_query_device()
120 		       dev->rdma_ctx);  in qedr_query_device()
121 		return -EINVAL;  in qedr_query_device()
126 	attr->fw_ver = qattr->fw_ver;  in qedr_query_device()
127 	attr->sys_image_guid = qattr->sys_image_guid;  in qedr_query_device()
128 	attr->max_mr_size = qattr->max_mr_size;  in qedr_query_device()
129 	attr->page_size_cap = qattr->page_size_caps;  in qedr_query_device()
130 	attr->vendor_id = qattr->vendor_id;  in qedr_query_device()
131 	attr->vendor_part_id = qattr->vendor_part_id;  in qedr_query_device()
132 	attr->hw_ver = qattr->hw_ver;  in qedr_query_device()
133 	attr->max_qp = qattr->max_qp;  in qedr_query_device()
134 	attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);  in qedr_query_device()
135 	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |  in qedr_query_device()
138 	attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;  in qedr_query_device()
140 	if (!rdma_protocol_iwarp(&dev->ibdev, 1))  in qedr_query_device()
141 		attr->device_cap_flags |= IB_DEVICE_XRC;  in qedr_query_device()
142 	attr->max_send_sge = qattr->max_sge;  in qedr_query_device()
143 	attr->max_recv_sge = qattr->max_sge;  in qedr_query_device()
144 	attr->max_sge_rd = qattr->max_sge;  in qedr_query_device()
145 	attr->max_cq = qattr->max_cq;  in qedr_query_device()
146 	attr->max_cqe = qattr->max_cqe;  in qedr_query_device()
147 	attr->max_mr = qattr->max_mr;  in qedr_query_device()
148 	attr->max_mw = qattr->max_mw;  in qedr_query_device()
149 	attr->max_pd = qattr->max_pd;  in qedr_query_device()
150 	attr->atomic_cap = dev->atomic_cap;  in qedr_query_device()
151 	attr->max_qp_init_rd_atom =  in qedr_query_device()
152 	    1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);  in qedr_query_device()
153 	attr->max_qp_rd_atom =  in qedr_query_device()
154 	    min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),  in qedr_query_device()
155 		attr->max_qp_init_rd_atom);  in qedr_query_device()
157 	attr->max_srq = qattr->max_srq;  in qedr_query_device()
158 	attr->max_srq_sge = qattr->max_srq_sge;  in qedr_query_device()
159 	attr->max_srq_wr = qattr->max_srq_wr;  in qedr_query_device()
161 	attr->local_ca_ack_delay = qattr->dev_ack_delay;  in qedr_query_device()
162 	attr->max_fast_reg_page_list_len = qattr->max_mr / 8;  in qedr_query_device()
163 	attr->max_pkeys = qattr->max_pkey;  in qedr_query_device()
164 	attr->max_ah = qattr->max_ah;  in qedr_query_device()
222 	if (!dev->rdma_ctx) {  in qedr_query_port()
224 		return -EINVAL;  in qedr_query_port()
227 	rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);  in qedr_query_port()
230 	if (rdma_port->port_state == QED_RDMA_PORT_UP) {  in qedr_query_port()
231 		attr->state = IB_PORT_ACTIVE;  in qedr_query_port()
232 		attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;  in qedr_query_port()
234 		attr->state = IB_PORT_DOWN;  in qedr_query_port()
235 		attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;  in qedr_query_port()
237 	attr->max_mtu = IB_MTU_4096;  in qedr_query_port()
238 	attr->lid = 0;  in qedr_query_port()
239 	attr->lmc = 0;  in qedr_query_port()
240 	attr->sm_lid = 0;  in qedr_query_port()
241 	attr->sm_sl = 0;  in qedr_query_port()
242 	attr->ip_gids = true;  in qedr_query_port()
243 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {  in qedr_query_port()
244 		attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);  in qedr_query_port()
245 		attr->gid_tbl_len = 1;  in qedr_query_port()
247 		attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);  in qedr_query_port()
248 		attr->gid_tbl_len = QEDR_MAX_SGID;  in qedr_query_port()
249 		attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;  in qedr_query_port()
251 	attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;  in qedr_query_port()
252 	attr->qkey_viol_cntr = 0;  in qedr_query_port()
253 	get_link_speed_and_width(rdma_port->link_speed,  in qedr_query_port()
254 				 &attr->active_speed, &attr->active_width);  in qedr_query_port()
255 	attr->max_msg_sz = rdma_port->max_msg_size;  in qedr_query_port()
256 	attr->max_vl_num = 4;  in qedr_query_port()
263 	struct ib_device *ibdev = uctx->device;  in qedr_alloc_ucontext()
273 		return -EFAULT;  in qedr_alloc_ucontext()
275 	if (udata->inlen) {  in qedr_alloc_ucontext()
277 					min(sizeof(ureq), udata->inlen));  in qedr_alloc_ucontext()
280 			return -EFAULT;  in qedr_alloc_ucontext()
282 		ctx->edpm_mode = !!(ureq.context_flags &  in qedr_alloc_ucontext()
284 		ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);  in qedr_alloc_ucontext()
287 	rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);  in qedr_alloc_ucontext()
295 	ctx->dpi = oparams.dpi;  in qedr_alloc_ucontext()
296 	ctx->dpi_addr = oparams.dpi_addr;  in qedr_alloc_ucontext()
297 	ctx->dpi_phys_addr = oparams.dpi_phys_addr;  in qedr_alloc_ucontext()
298 	ctx->dpi_size = oparams.dpi_size;  in qedr_alloc_ucontext()
301 		rc = -ENOMEM;  in qedr_alloc_ucontext()
305 	entry->io_address = ctx->dpi_phys_addr;  in qedr_alloc_ucontext()
306 	entry->length = ctx->dpi_size;  in qedr_alloc_ucontext()
307 	entry->mmap_flag = QEDR_USER_MMAP_IO_WC;  in qedr_alloc_ucontext()
308 	entry->dpi = ctx->dpi;  in qedr_alloc_ucontext()
309 	entry->dev = dev;  in qedr_alloc_ucontext()
310 	rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,  in qedr_alloc_ucontext()
311 					 ctx->dpi_size);  in qedr_alloc_ucontext()
316 	ctx->db_mmap_entry = &entry->rdma_entry;  in qedr_alloc_ucontext()
318 	if (!dev->user_dpm_enabled)  in qedr_alloc_ucontext()
320 	else if (rdma_protocol_iwarp(&dev->ibdev, 1))  in qedr_alloc_ucontext()
336 	uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);  in qedr_alloc_ucontext()
337 	uresp.db_size = ctx->dpi_size;  in qedr_alloc_ucontext()
338 	uresp.max_send_wr = dev->attr.max_sqe;  in qedr_alloc_ucontext()
339 	uresp.max_recv_wr = dev->attr.max_rqe;  in qedr_alloc_ucontext()
340 	uresp.max_srq_wr = dev->attr.max_srq_wr;  in qedr_alloc_ucontext()
343 	uresp.sges_per_srq_wr = dev->attr.max_srq_sge;  in qedr_alloc_ucontext()
350 	ctx->dev = dev;  in qedr_alloc_ucontext()
353 		 &ctx->ibucontext);  in qedr_alloc_ucontext()
357 	if (!ctx->db_mmap_entry)  in qedr_alloc_ucontext()
358 		dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);  in qedr_alloc_ucontext()
360 		rdma_user_mmap_entry_remove(ctx->db_mmap_entry);  in qedr_alloc_ucontext()
369 	DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",  in qedr_dealloc_ucontext()
372 	rdma_user_mmap_entry_remove(uctx->db_mmap_entry);  in qedr_dealloc_ucontext()
378 	struct qedr_dev *dev = entry->dev;  in qedr_mmap_free()
380 	if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)  in qedr_mmap_free()
381 		free_page((unsigned long)entry->address);  in qedr_mmap_free()
382 	else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)  in qedr_mmap_free()
383 		dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);  in qedr_mmap_free()
390 	struct ib_device *dev = ucontext->device;  in qedr_mmap()
391 	size_t length = vma->vm_end - vma->vm_start;  in qedr_mmap()
399 		  vma->vm_start, vma->vm_end, length, vma->vm_pgoff);  in qedr_mmap()
404 			  vma->vm_pgoff);  in qedr_mmap()
405 		return -EINVAL;  in qedr_mmap()
410 		  entry->io_address, length, entry->mmap_flag);  in qedr_mmap()
412 	switch (entry->mmap_flag) {  in qedr_mmap()
414 		pfn = entry->io_address >> PAGE_SHIFT;  in qedr_mmap()
416 				       pgprot_writecombine(vma->vm_page_prot),  in qedr_mmap()
420 		rc = vm_insert_page(vma, vma->vm_start,  in qedr_mmap()
421 				    virt_to_page(entry->address));  in qedr_mmap()
424 		rc = -EINVAL;  in qedr_mmap()
430 			  entry->io_address, length, entry->mmap_flag, rc);  in qedr_mmap()
438 	struct ib_device *ibdev = ibpd->device;  in qedr_alloc_pd()
447 	if (!dev->rdma_ctx) {  in qedr_alloc_pd()
449 		return -EINVAL;  in qedr_alloc_pd()
452 	rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);  in qedr_alloc_pd()
456 	pd->pd_id = pd_id;  in qedr_alloc_pd()
468 			dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);  in qedr_alloc_pd()
472 		pd->uctx = context;  in qedr_alloc_pd()
473 		pd->uctx->pd = pd;  in qedr_alloc_pd()
481 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);  in qedr_dealloc_pd()
484 	DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);  in qedr_dealloc_pd()
485 	dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);  in qedr_dealloc_pd()
492 	struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);  in qedr_alloc_xrcd()
495 	return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);  in qedr_alloc_xrcd()
500 	struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);  in qedr_dealloc_xrcd()
501 	u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;  in qedr_dealloc_xrcd()
503 	dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);  in qedr_dealloc_xrcd()
509 	struct pci_dev *pdev = dev->pdev;  in qedr_free_pbl()
512 	for (i = 0; i < pbl_info->num_pbls; i++) {  in qedr_free_pbl()
515 		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,  in qedr_free_pbl()
533 	struct pci_dev *pdev = dev->pdev;  in qedr_alloc_pbl_tbl()
540 	pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);  in qedr_alloc_pbl_tbl()
542 		return ERR_PTR(-ENOMEM);  in qedr_alloc_pbl_tbl()
544 	for (i = 0; i < pbl_info->num_pbls; i++) {  in qedr_alloc_pbl_tbl()
545 		va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,  in qedr_alloc_pbl_tbl()
554 	/* Two-Layer PBLs, if we have more than one pbl we need to initialize  in qedr_alloc_pbl_tbl()
558 	for (i = 0; i < pbl_info->num_pbls - 1; i++)  in qedr_alloc_pbl_tbl()
564 	for (i--; i >= 0; i--)  in qedr_alloc_pbl_tbl()
565 		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,  in qedr_alloc_pbl_tbl()
570 	return ERR_PTR(-ENOMEM);  in qedr_alloc_pbl_tbl()
585 			return -EINVAL;  in qedr_prepare_pbl_tbl()
601 		pbl_info->two_layered = true;  in qedr_prepare_pbl_tbl()
607 		pbl_info->two_layered = false;  in qedr_prepare_pbl_tbl()
610 	pbl_info->num_pbls = num_pbls;  in qedr_prepare_pbl_tbl()
611 	pbl_info->pbl_size = pbl_size;  in qedr_prepare_pbl_tbl()
612 	pbl_info->num_pbes = num_pbes;  in qedr_prepare_pbl_tbl()
616 		 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);  in qedr_prepare_pbl_tbl()
630 	if (!pbl_info->num_pbes)  in qedr_populate_pbls()
636 	if (pbl_info->two_layered)  in qedr_populate_pbls()
641 	pbe = (struct regpair *)pbl_tbl->va;  in qedr_populate_pbls()
652 		pbe->lo = cpu_to_le32(pg_addr);  in qedr_populate_pbls()
653 		pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));  in qedr_populate_pbls()
659 		if (total_num_pbes == pbl_info->num_pbes)  in qedr_populate_pbls()
664 		if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {  in qedr_populate_pbls()
666 			pbe = (struct regpair *)pbl_tbl->va;  in qedr_populate_pbls()
683 	return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,  in qedr_db_recovery_add()
699 	dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);  in qedr_db_recovery_del()
712 	uresp.icid = cq->icid;  in qedr_copy_cq_uresp()
713 	if (cq->q.db_mmap_entry)  in qedr_copy_cq_uresp()
715 			rdma_user_mmap_get_offset(cq->q.db_mmap_entry);  in qedr_copy_cq_uresp()
719 		DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);  in qedr_copy_cq_uresp()
726 	if (cq->latest_cqe == cq->toggle_cqe)  in consume_cqe()
727 		cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;  in consume_cqe()
729 	cq->latest_cqe = qed_chain_consume(&cq->pbl);  in consume_cqe()
753 	/* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */  in qedr_init_user_db_rec()
754 	if (requires_db_rec == 0 || !uctx->db_rec)  in qedr_init_user_db_rec()
758 	q->db_rec_data = (void *)get_zeroed_page(GFP_USER);  in qedr_init_user_db_rec()
759 	if (!q->db_rec_data) {  in qedr_init_user_db_rec()
761 		return -ENOMEM;  in qedr_init_user_db_rec()
768 	entry->address = q->db_rec_data;  in qedr_init_user_db_rec()
769 	entry->length = PAGE_SIZE;  in qedr_init_user_db_rec()
770 	entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;  in qedr_init_user_db_rec()
771 	rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,  in qedr_init_user_db_rec()
772 					 &entry->rdma_entry,  in qedr_init_user_db_rec()
777 	q->db_mmap_entry = &entry->rdma_entry;  in qedr_init_user_db_rec()
785 	free_page((unsigned long)q->db_rec_data);  in qedr_init_user_db_rec()
786 	q->db_rec_data = NULL;  in qedr_init_user_db_rec()
787 	return -ENOMEM;  in qedr_init_user_db_rec()
800 	q->buf_addr = buf_addr;  in qedr_init_user_queue()
801 	q->buf_len = buf_len;  in qedr_init_user_queue()
802 	q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);  in qedr_init_user_queue()
803 	if (IS_ERR(q->umem)) {  in qedr_init_user_queue()
805 		       PTR_ERR(q->umem));  in qedr_init_user_queue()
806 		return PTR_ERR(q->umem);  in qedr_init_user_queue()
809 	fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);  in qedr_init_user_queue()
810 	rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);  in qedr_init_user_queue()
815 		q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);  in qedr_init_user_queue()
816 		if (IS_ERR(q->pbl_tbl)) {  in qedr_init_user_queue()
817 			rc = PTR_ERR(q->pbl_tbl);  in qedr_init_user_queue()
820 		qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,  in qedr_init_user_queue()
823 		q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);  in qedr_init_user_queue()
824 		if (!q->pbl_tbl) {  in qedr_init_user_queue()
825 			rc = -ENOMEM;  in qedr_init_user_queue()
834 	ib_umem_release(q->umem);  in qedr_init_user_queue()
835 	q->umem = NULL;  in qedr_init_user_queue()
849 	params->cq_handle_hi = upper_32_bits((uintptr_t)cq);  in qedr_init_cq_params()
850 	params->cq_handle_lo = lower_32_bits((uintptr_t)cq);  in qedr_init_cq_params()
851 	params->cnq_id = vector;  in qedr_init_cq_params()
852 	params->cq_size = chain_entries - 1;  in qedr_init_cq_params()
853 	params->dpi = (ctx) ? ctx->dpi : dev->dpi;  in qedr_init_cq_params()
854 	params->pbl_num_pages = page_cnt;  in qedr_init_cq_params()
855 	params->pbl_ptr = pbl_ptr;  in qedr_init_cq_params()
856 	params->pbl_two_level = 0;  in qedr_init_cq_params()
861 	cq->db.data.agg_flags = flags;  in doorbell_cq()
862 	cq->db.data.value = cpu_to_le32(cons);  in doorbell_cq()
863 	writeq(cq->db.raw, cq->db_addr);  in doorbell_cq()
872 	dev = get_qedr_dev(ibcq->device);  in qedr_arm_cq()
874 	if (cq->destroyed) {  in qedr_arm_cq()
876 		       "warning: arm was invoked after destroy for cq %p (icid=%d)\n",  in qedr_arm_cq()
877 		       cq, cq->icid);  in qedr_arm_cq()
878 		return -EINVAL;  in qedr_arm_cq()
882 	if (cq->cq_type == QEDR_CQ_TYPE_GSI)  in qedr_arm_cq()
885 	spin_lock_irqsave(&cq->cq_lock, sflags);  in qedr_arm_cq()
887 	cq->arm_flags = 0;  in qedr_arm_cq()
890 		cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;  in qedr_arm_cq()
893 		cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;  in qedr_arm_cq()
895 	doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);  in qedr_arm_cq()
897 	spin_unlock_irqrestore(&cq->cq_lock, sflags);  in qedr_arm_cq()
905 	struct ib_udata *udata = &attrs->driver_udata;  in qedr_create_cq()
906 	struct ib_device *ibdev = ibcq->device;  in qedr_create_cq()
920 	int vector = attr->comp_vector;  in qedr_create_cq()
921 	int entries = attr->cqe;  in qedr_create_cq()
927 	u16 icid;  in qedr_create_cq()  local
934 	if (attr->flags)  in qedr_create_cq()
935 		return -EOPNOTSUPP;  in qedr_create_cq()
941 		return -EINVAL;  in qedr_create_cq()
948 	/* calc db offset. user will add DPI base, kernel will add db addr */  in qedr_create_cq()
953 							 udata->inlen))) {  in qedr_create_cq()
965 		cq->cq_type = QEDR_CQ_TYPE_USER;  in qedr_create_cq()
967 		rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,  in qedr_create_cq()
973 		pbl_ptr = cq->q.pbl_tbl->pa;  in qedr_create_cq()
974 		page_cnt = cq->q.pbl_info.num_pbes;  in qedr_create_cq()
976 		cq->ibcq.cqe = chain_entries;  in qedr_create_cq()
977 		cq->q.db_addr = ctx->dpi_addr + db_offset;  in qedr_create_cq()
979 		cq->cq_type = QEDR_CQ_TYPE_KERNEL;  in qedr_create_cq()
981 		rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,  in qedr_create_cq()
986 		page_cnt = qed_chain_get_page_cnt(&cq->pbl);  in qedr_create_cq()
987 		pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);  in qedr_create_cq()
988 		cq->ibcq.cqe = cq->pbl.capacity;  in qedr_create_cq()
994 	rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);  in qedr_create_cq()
998 	cq->icid = icid;  in qedr_create_cq()
999 	cq->sig = QEDR_CQ_MAGIC_NUMBER;  in qedr_create_cq()
1000 	spin_lock_init(&cq->cq_lock);  in qedr_create_cq()
1007 		rc = qedr_db_recovery_add(dev, cq->q.db_addr,  in qedr_create_cq()
1008 					  &cq->q.db_rec_data->db_data,  in qedr_create_cq()
1016 		cq->db.data.icid = cq->icid;  in qedr_create_cq()
1017 		cq->db_addr = dev->db_addr + db_offset;  in qedr_create_cq()
1018 		cq->db.data.params = DB_AGG_CMD_MAX <<  in qedr_create_cq()
1022 		cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);  in qedr_create_cq()
1023 		cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;  in qedr_create_cq()
1024 		cq->latest_cqe = NULL;  in qedr_create_cq()
1026 		cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);  in qedr_create_cq()
1028 		rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,  in qedr_create_cq()
1035 		 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",  in qedr_create_cq()
1036 		 cq->icid, cq, params.cq_size);  in qedr_create_cq()
1041 	destroy_iparams.icid = cq->icid;  in qedr_create_cq()
1042 	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,  in qedr_create_cq()
1046 		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);  in qedr_create_cq()
1047 		ib_umem_release(cq->q.umem);  in qedr_create_cq()
1048 		if (cq->q.db_mmap_entry)  in qedr_create_cq()
1049 			rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);  in qedr_create_cq()
1051 		dev->ops->common->chain_free(dev->cdev, &cq->pbl);  in qedr_create_cq()
1054 	return -EINVAL;  in qedr_create_cq()
1062 	struct qedr_dev *dev = get_qedr_dev(ibcq->device);  in qedr_destroy_cq()
1068 	DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);  in qedr_destroy_cq()
1070 	cq->destroyed = 1;  in qedr_destroy_cq()
1073 	if (cq->cq_type == QEDR_CQ_TYPE_GSI) {  in qedr_destroy_cq()
1074 		qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);  in qedr_destroy_cq()
1078 	iparams.icid = cq->icid;  in qedr_destroy_cq()
1079 	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);  in qedr_destroy_cq()
1080 	dev->ops->common->chain_free(dev->cdev, &cq->pbl);  in qedr_destroy_cq()
1083 		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);  in qedr_destroy_cq()
1084 		ib_umem_release(cq->q.umem);  in qedr_destroy_cq()
1086 		if (cq->q.db_rec_data) {  in qedr_destroy_cq()
1087 			qedr_db_recovery_del(dev, cq->q.db_addr,  in qedr_destroy_cq()
1088 					     &cq->q.db_rec_data->db_data);  in qedr_destroy_cq()
1089 			rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);  in qedr_destroy_cq()
1092 		qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);  in qedr_destroy_cq()
1095 	/* We don't want the IRQ handler to handle a non-existing CQ so we  in qedr_destroy_cq()
1107 	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {  in qedr_destroy_cq()
1109 		iter--;  in qedr_destroy_cq()
1113 	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {  in qedr_destroy_cq()
1115 		iter--;  in qedr_destroy_cq()
1134 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);  in get_gid_info_from_table()
1139 	gid_attr = grh->sgid_attr;  in get_gid_info_from_table()
1140 	ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);  in get_gid_info_from_table()
1147 		memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],  in get_gid_info_from_table()
1148 		       sizeof(qp_params->sgid));  in get_gid_info_from_table()
1149 		memcpy(&qp_params->dgid.bytes[0],  in get_gid_info_from_table()
1150 		       &grh->dgid,  in get_gid_info_from_table()
1151 		       sizeof(qp_params->dgid));  in get_gid_info_from_table()
1152 		qp_params->roce_mode = ROCE_V2_IPV6;  in get_gid_info_from_table()
1153 		SET_FIELD(qp_params->modify_flags,  in get_gid_info_from_table()
1157 		memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],  in get_gid_info_from_table()
1158 		       sizeof(qp_params->sgid));  in get_gid_info_from_table()
1159 		memcpy(&qp_params->dgid.bytes[0],  in get_gid_info_from_table()
1160 		       &grh->dgid,  in get_gid_info_from_table()
1161 		       sizeof(qp_params->dgid));  in get_gid_info_from_table()
1162 		qp_params->roce_mode = ROCE_V1;  in get_gid_info_from_table()
1165 		memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));  in get_gid_info_from_table()
1166 		memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));  in get_gid_info_from_table()
1167 		ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);  in get_gid_info_from_table()
1168 		qp_params->sgid.ipv4_addr = ipv4_addr;  in get_gid_info_from_table()
1170 		    qedr_get_ipv4_from_gid(grh->dgid.raw);  in get_gid_info_from_table()
1171 		qp_params->dgid.ipv4_addr = ipv4_addr;  in get_gid_info_from_table()
1172 		SET_FIELD(qp_params->modify_flags,  in get_gid_info_from_table()
1174 		qp_params->roce_mode = ROCE_V2_IPV4;  in get_gid_info_from_table()
1177 		return -EINVAL;  in get_gid_info_from_table()
1181 		qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);  in get_gid_info_from_table()
1182 		qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);  in get_gid_info_from_table()
1185 	if (qp_params->vlan_id >= VLAN_CFI_MASK)  in get_gid_info_from_table()
1186 		qp_params->vlan_id = 0;  in get_gid_info_from_table()
1195 	struct qedr_device_attr *qattr = &dev->attr;  in qedr_check_qp_attrs()
1197 	/* QP0... attrs->qp_type == IB_QPT_GSI */  in qedr_check_qp_attrs()
1198 	if (attrs->qp_type != IB_QPT_RC &&  in qedr_check_qp_attrs()
1199 	    attrs->qp_type != IB_QPT_GSI &&  in qedr_check_qp_attrs()
1200 	    attrs->qp_type != IB_QPT_XRC_INI &&  in qedr_check_qp_attrs()
1201 	    attrs->qp_type != IB_QPT_XRC_TGT) {  in qedr_check_qp_attrs()
1204 			 attrs->qp_type);  in qedr_check_qp_attrs()
1205 		return -EOPNOTSUPP;  in qedr_check_qp_attrs()
1208 	if (attrs->cap.max_send_wr > qattr->max_sqe) {  in qedr_check_qp_attrs()
1211 		       attrs->cap.max_send_wr, qattr->max_sqe);  in qedr_check_qp_attrs()
1212 		return -EINVAL;  in qedr_check_qp_attrs()
1215 	if (attrs->cap.max_inline_data > qattr->max_inline) {  in qedr_check_qp_attrs()
1218 		       attrs->cap.max_inline_data, qattr->max_inline);  in qedr_check_qp_attrs()
1219 		return -EINVAL;  in qedr_check_qp_attrs()
1222 	if (attrs->cap.max_send_sge > qattr->max_sge) {  in qedr_check_qp_attrs()
1225 		       attrs->cap.max_send_sge, qattr->max_sge);  in qedr_check_qp_attrs()
1226 		return -EINVAL;  in qedr_check_qp_attrs()
1229 	if (attrs->cap.max_recv_sge > qattr->max_sge) {  in qedr_check_qp_attrs()
1232 		       attrs->cap.max_recv_sge, qattr->max_sge);  in qedr_check_qp_attrs()
1233 		return -EINVAL;  in qedr_check_qp_attrs()
1239 	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&  in qedr_check_qp_attrs()
1240 	    (attrs->qp_type != IB_QPT_XRC_TGT) &&  in qedr_check_qp_attrs()
1241 	    (attrs->qp_type != IB_QPT_XRC_INI)) {  in qedr_check_qp_attrs()
1242 		struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);  in qedr_check_qp_attrs()
1243 		struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);  in qedr_check_qp_attrs()
1245 		if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||  in qedr_check_qp_attrs()
1246 		    (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {  in qedr_check_qp_attrs()
1249 			return -EINVAL;  in qedr_check_qp_attrs()
1262 	uresp.srq_id = srq->srq_id;  in qedr_copy_srq_uresp()
1276 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {  in qedr_copy_rq_uresp()
1277 		uresp->rq_db_offset =  in qedr_copy_rq_uresp()
1279 		uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);  in qedr_copy_rq_uresp()
1281 		uresp->rq_db_offset =  in qedr_copy_rq_uresp()
1285 	uresp->rq_icid = qp->icid;  in qedr_copy_rq_uresp()
1286 	if (qp->urq.db_mmap_entry)  in qedr_copy_rq_uresp()
1287 		uresp->rq_db_rec_addr =  in qedr_copy_rq_uresp()
1288 			rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);  in qedr_copy_rq_uresp()
1295 	uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);  in qedr_copy_sq_uresp()
1298 	if (rdma_protocol_iwarp(&dev->ibdev, 1))  in qedr_copy_sq_uresp()
1299 		uresp->sq_icid = qp->icid;  in qedr_copy_sq_uresp()
1301 		uresp->sq_icid = qp->icid + 1;  in qedr_copy_sq_uresp()
1303 	if (qp->usq.db_mmap_entry)  in qedr_copy_sq_uresp()
1304 		uresp->sq_db_rec_addr =  in qedr_copy_sq_uresp()
1305 			rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);  in qedr_copy_sq_uresp()
1322 	uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;  in qedr_copy_qp_uresp()
1323 	uresp->qp_id = qp->qp_id;  in qedr_copy_qp_uresp()
1328 		       "create qp: failed a copy to user space with qp icid=0x%x.\n",  in qedr_copy_qp_uresp()
1329 		       qp->icid);  in qedr_copy_qp_uresp()
1336 	qed_chain_reset(&qph->pbl);  in qedr_reset_qp_hwq_info()
1337 	qph->prod = 0;  in qedr_reset_qp_hwq_info()
1338 	qph->cons = 0;  in qedr_reset_qp_hwq_info()
1339 	qph->wqe_cons = 0;  in qedr_reset_qp_hwq_info()
1340 	qph->db_data.data.value = cpu_to_le16(0);  in qedr_reset_qp_hwq_info()
1348 	spin_lock_init(&qp->q_lock);  in qedr_set_common_qp_params()
1349 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {  in qedr_set_common_qp_params()
1350 		kref_init(&qp->refcnt);  in qedr_set_common_qp_params()
1351 		init_completion(&qp->iwarp_cm_comp);  in qedr_set_common_qp_params()
1352 		init_completion(&qp->qp_rel_comp);  in qedr_set_common_qp_params()
1355 	qp->pd = pd;  in qedr_set_common_qp_params()
1356 	qp->qp_type = attrs->qp_type;  in qedr_set_common_qp_params()
1357 	qp->max_inline_data = attrs->cap.max_inline_data;  in qedr_set_common_qp_params()
1358 	qp->state = QED_ROCE_QP_STATE_RESET;  in qedr_set_common_qp_params()
1360 	qp->prev_wqe_size = 0;  in qedr_set_common_qp_params()
1362 	qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;  in qedr_set_common_qp_params()
1363 	qp->dev = dev;  in qedr_set_common_qp_params()
1365 		qedr_reset_qp_hwq_info(&qp->sq);  in qedr_set_common_qp_params()
1366 		qp->sq.max_sges = attrs->cap.max_send_sge;  in qedr_set_common_qp_params()
1367 		qp->sq_cq = get_qedr_cq(attrs->send_cq);  in qedr_set_common_qp_params()
1370 			 qp->sq.max_sges, qp->sq_cq->icid);  in qedr_set_common_qp_params()
1373 	if (attrs->srq)  in qedr_set_common_qp_params()
1374 		qp->srq = get_qedr_srq(attrs->srq);  in qedr_set_common_qp_params()
1377 		qedr_reset_qp_hwq_info(&qp->rq);  in qedr_set_common_qp_params()
1378 		qp->rq_cq = get_qedr_cq(attrs->recv_cq);  in qedr_set_common_qp_params()
1379 		qp->rq.max_sges = attrs->cap.max_recv_sge;  in qedr_set_common_qp_params()
1382 			 qp->rq.max_sges, qp->rq_cq->icid);  in qedr_set_common_qp_params()
1387 		 pd->pd_id, qp->qp_type, qp->max_inline_data,  in qedr_set_common_qp_params()
1388 		 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);  in qedr_set_common_qp_params()
1391 		 qp->sq.max_sges, qp->sq_cq->icid);  in qedr_set_common_qp_params()
1399 		qp->sq.db = dev->db_addr +  in qedr_set_roce_db_info()
1401 		qp->sq.db_data.data.icid = qp->icid + 1;  in qedr_set_roce_db_info()
1402 		rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,  in qedr_set_roce_db_info()
1409 		qp->rq.db = dev->db_addr +  in qedr_set_roce_db_info()
1411 		qp->rq.db_data.data.icid = qp->icid;  in qedr_set_roce_db_info()
1412 		rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,  in qedr_set_roce_db_info()
1415 			qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);  in qedr_set_roce_db_info()
1425 	struct qedr_device_attr *qattr = &dev->attr;  in qedr_check_srq_params()
1427 	if (attrs->attr.max_wr > qattr->max_srq_wr) {  in qedr_check_srq_params()
1430 		       attrs->attr.max_wr, qattr->max_srq_wr);  in qedr_check_srq_params()
1431 		return -EINVAL;  in qedr_check_srq_params()
1434 	if (attrs->attr.max_sge > qattr->max_sge) {  in qedr_check_srq_params()
1437 		       attrs->attr.max_sge, qattr->max_sge);  in qedr_check_srq_params()
1440 	if (!udata && attrs->srq_type == IB_SRQT_XRC) {  in qedr_check_srq_params()
1441 		DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");  in qedr_check_srq_params()
1442 		return -EINVAL;  in qedr_check_srq_params()
1450 	qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);  in qedr_free_srq_user_params()
1451 	ib_umem_release(srq->usrq.umem);  in qedr_free_srq_user_params()
1452 	ib_umem_release(srq->prod_umem);  in qedr_free_srq_user_params()
1457 	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;  in qedr_free_srq_kernel_params()
1458 	struct qedr_dev *dev = srq->dev;  in qedr_free_srq_kernel_params()
1460 	dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);  in qedr_free_srq_kernel_params()
1462 	dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),  in qedr_free_srq_kernel_params()
1463 			  hw_srq->virt_prod_pair_addr,  in qedr_free_srq_kernel_params()
1464 			  hw_srq->phy_prod_pair_addr);  in qedr_free_srq_kernel_params()
1475 	rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,  in qedr_init_srq_user_params()
1476 				  ureq->srq_len, false, access, 1);  in qedr_init_srq_user_params()
1480 	srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,  in qedr_init_srq_user_params()
1482 	if (IS_ERR(srq->prod_umem)) {  in qedr_init_srq_user_params()
1483 		qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);  in qedr_init_srq_user_params()
1484 		ib_umem_release(srq->usrq.umem);  in qedr_init_srq_user_params()
1485 		DP_ERR(srq->dev,  in qedr_init_srq_user_params()
1487 		       PTR_ERR(srq->prod_umem));  in qedr_init_srq_user_params()
1488 		return PTR_ERR(srq->prod_umem);  in qedr_init_srq_user_params()
1491 	sg = srq->prod_umem->sgt_append.sgt.sgl;  in qedr_init_srq_user_params()
1492 	srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);  in qedr_init_srq_user_params()
1501 	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;  in qedr_alloc_srq_kernel_params()
1513 	va = dma_alloc_coherent(&dev->pdev->dev,  in qedr_alloc_srq_kernel_params()
1519 		return -ENOMEM;  in qedr_alloc_srq_kernel_params()
1522 	hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;  in qedr_alloc_srq_kernel_params()
1523 	hw_srq->virt_prod_pair_addr = va;  in qedr_alloc_srq_kernel_params()
1525 	num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;  in qedr_alloc_srq_kernel_params()
1528 	rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);  in qedr_alloc_srq_kernel_params()
1532 	hw_srq->num_elems = num_elems;  in qedr_alloc_srq_kernel_params()
1537 	dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),  in qedr_alloc_srq_kernel_params()
1547 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);  in qedr_create_srq()
1549 	struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);  in qedr_create_srq()
1561 	if (init_attr->srq_type != IB_SRQT_BASIC &&  in qedr_create_srq()
1562 	    init_attr->srq_type != IB_SRQT_XRC)  in qedr_create_srq()
1563 		return -EOPNOTSUPP;  in qedr_create_srq()
1567 		return -EINVAL;  in qedr_create_srq()
1569 	srq->dev = dev;  in qedr_create_srq()
1570 	srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);  in qedr_create_srq()
1571 	hw_srq = &srq->hw_srq;  in qedr_create_srq()
1572 	spin_lock_init(&srq->lock);  in qedr_create_srq()
1574 	hw_srq->max_wr = init_attr->attr.max_wr;  in qedr_create_srq()
1575 	hw_srq->max_sges = init_attr->attr.max_sge;  in qedr_create_srq()
1579 							 udata->inlen))) {  in qedr_create_srq()
1589 		page_cnt = srq->usrq.pbl_info.num_pbes;  in qedr_create_srq()
1590 		pbl_base_addr = srq->usrq.pbl_tbl->pa;  in qedr_create_srq()
1591 		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;  in qedr_create_srq()
1600 		pbl = &hw_srq->pbl;  in qedr_create_srq()
1603 		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;  in qedr_create_srq()
1607 	in_params.pd_id = pd->pd_id;  in qedr_create_srq()
1612 	if (srq->is_xrc) {  in qedr_create_srq()
1613 		struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);  in qedr_create_srq()
1614 		struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);  in qedr_create_srq()
1617 		in_params.xrcd_id = xrcd->xrcd_id;  in qedr_create_srq()
1618 		in_params.cq_cid = cq->icid;  in qedr_create_srq()
1621 	rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);  in qedr_create_srq()
1625 	srq->srq_id = out_params.srq_id;  in qedr_create_srq()
1633 	rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);  in qedr_create_srq()
1638 		 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);  in qedr_create_srq()
1642 	destroy_in_params.srq_id = srq->srq_id;  in qedr_create_srq()
1644 	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);  in qedr_create_srq()
1651 	return -EFAULT;  in qedr_create_srq()
1657 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);  in qedr_destroy_srq()
1660 	xa_erase_irq(&dev->srqs, srq->srq_id);  in qedr_destroy_srq()
1661 	in_params.srq_id = srq->srq_id;  in qedr_destroy_srq()
1662 	in_params.is_xrc = srq->is_xrc;  in qedr_destroy_srq()
1663 	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);  in qedr_destroy_srq()
1665 	if (ibsrq->uobject)  in qedr_destroy_srq()
1672 		 srq->srq_id);  in qedr_destroy_srq()
1680 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);  in qedr_modify_srq()
1688 		return -EINVAL;  in qedr_modify_srq()
1692 		if (attr->srq_limit >= srq->hw_srq.max_wr) {  in qedr_modify_srq()
1695 			       attr->srq_limit, srq->hw_srq.max_wr);  in qedr_modify_srq()
1696 			return -EINVAL;  in qedr_modify_srq()
1699 		in_params.srq_id = srq->srq_id;  in qedr_modify_srq()
1700 		in_params.wqe_limit = attr->srq_limit;  in qedr_modify_srq()
1701 		rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);  in qedr_modify_srq()
1706 	srq->srq_limit = attr->srq_limit;  in qedr_modify_srq()
1709 		 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);  in qedr_modify_srq()
1737 	params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);  in qedr_init_common_qp_in_params()
1738 	params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);  in qedr_init_common_qp_in_params()
1740 	params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);  in qedr_init_common_qp_in_params()
1741 	params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;  in qedr_init_common_qp_in_params()
1742 	params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);  in qedr_init_common_qp_in_params()
1743 	params->stats_queue = 0;  in qedr_init_common_qp_in_params()
1746 		params->pd = pd->pd_id;  in qedr_init_common_qp_in_params()
1747 		params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;  in qedr_init_common_qp_in_params()
1751 		params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;  in qedr_init_common_qp_in_params()
1754 		params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;  in qedr_init_common_qp_in_params()
1757 		params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;  in qedr_init_common_qp_in_params()
1758 		params->srq_id = qp->srq->srq_id;  in qedr_init_common_qp_in_params()
1759 		params->use_srq = true;  in qedr_init_common_qp_in_params()
1761 		params->srq_id = 0;  in qedr_init_common_qp_in_params()
1762 		params->use_srq = false;  in qedr_init_common_qp_in_params()
1776 		 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,  in qedr_qp_user_print()
1777 		 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,  in qedr_qp_user_print()
1778 		 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,  in qedr_qp_user_print()
1779 		 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);  in qedr_qp_user_print()
1787 	qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;  in qedr_iwarp_populate_user_qp()
1788 	qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;  in qedr_iwarp_populate_user_qp()
1790 	qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,  in qedr_iwarp_populate_user_qp()
1791 			   &qp->usq.pbl_info, FW_PAGE_SHIFT);  in qedr_iwarp_populate_user_qp()
1792 	if (!qp->srq) {  in qedr_iwarp_populate_user_qp()
1793 		qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;  in qedr_iwarp_populate_user_qp()
1794 		qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;  in qedr_iwarp_populate_user_qp()
1797 	qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,  in qedr_iwarp_populate_user_qp()
1798 			   &qp->urq.pbl_info, FW_PAGE_SHIFT);  in qedr_iwarp_populate_user_qp()
1806 		ib_umem_release(qp->usq.umem);  in qedr_cleanup_user()
1807 		qp->usq.umem = NULL;  in qedr_cleanup_user()
1811 		ib_umem_release(qp->urq.umem);  in qedr_cleanup_user()
1812 		qp->urq.umem = NULL;  in qedr_cleanup_user()
1815 	if (rdma_protocol_roce(&dev->ibdev, 1)) {  in qedr_cleanup_user()
1816 		qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);  in qedr_cleanup_user()
1817 		qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);  in qedr_cleanup_user()
1819 		kfree(qp->usq.pbl_tbl);  in qedr_cleanup_user()
1820 		kfree(qp->urq.pbl_tbl);  in qedr_cleanup_user()
1823 	if (qp->usq.db_rec_data) {  in qedr_cleanup_user()
1824 		qedr_db_recovery_del(dev, qp->usq.db_addr,  in qedr_cleanup_user()
1825 				     &qp->usq.db_rec_data->db_data);  in qedr_cleanup_user()
1826 		rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);  in qedr_cleanup_user()
1829 	if (qp->urq.db_rec_data) {  in qedr_cleanup_user()
1830 		qedr_db_recovery_del(dev, qp->urq.db_addr,  in qedr_cleanup_user()
1831 				     &qp->urq.db_rec_data->db_data);  in qedr_cleanup_user()
1832 		rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);  in qedr_cleanup_user()
1835 	if (rdma_protocol_iwarp(&dev->ibdev, 1))  in qedr_cleanup_user()
1836 		qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,  in qedr_cleanup_user()
1837 				     &qp->urq.db_rec_db2_data);  in qedr_cleanup_user()
1850 	int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);  in qedr_create_user_qp()
1855 	qp->create_type = QEDR_QP_CREATE_USER;  in qedr_create_user_qp()
1859 		ctx = pd->uctx;  in qedr_create_user_qp()
1864 					udata->inlen));  in qedr_create_user_qp()
1872 		/* SQ - read access only (0) */  in qedr_create_user_qp()
1873 		rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,  in qedr_create_user_qp()
1880 		/* RQ - read access only (0) */  in qedr_create_user_qp()
1881 		rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,  in qedr_create_user_qp()
1884 			ib_umem_release(qp->usq.umem);  in qedr_create_user_qp()
1885 			qp->usq.umem = NULL;  in qedr_create_user_qp()
1886 			if (rdma_protocol_roce(&dev->ibdev, 1)) {  in qedr_create_user_qp()
1887 				qedr_free_pbl(dev, &qp->usq.pbl_info,  in qedr_create_user_qp()
1888 					      qp->usq.pbl_tbl);  in qedr_create_user_qp()
1890 				kfree(qp->usq.pbl_tbl);  in qedr_create_user_qp()
1901 	if (qp->qp_type == IB_QPT_XRC_TGT) {  in qedr_create_user_qp()
1902 		struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);  in qedr_create_user_qp()
1904 		in_params.xrcd_id = xrcd->xrcd_id;  in qedr_create_user_qp()
1905 		in_params.qp_handle_lo = qp->qp_id;  in qedr_create_user_qp()
1910 		in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;  in qedr_create_user_qp()
1911 		in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;  in qedr_create_user_qp()
1915 		in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;  in qedr_create_user_qp()
1916 		in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;  in qedr_create_user_qp()
1920 		SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);  in qedr_create_user_qp()
1922 	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,  in qedr_create_user_qp()
1925 	if (!qp->qed_qp) {  in qedr_create_user_qp()
1926 		rc = -ENOMEM;  in qedr_create_user_qp()
1930 	if (rdma_protocol_iwarp(&dev->ibdev, 1))  in qedr_create_user_qp()
1933 	qp->qp_id = out_params.qp_id;  in qedr_create_user_qp()
1934 	qp->icid = out_params.icid;  in qedr_create_user_qp()
1944 		qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;  in qedr_create_user_qp()
1945 		qp->sq.max_wr = attrs->cap.max_send_wr;  in qedr_create_user_qp()
1946 		rc = qedr_db_recovery_add(dev, qp->usq.db_addr,  in qedr_create_user_qp()
1947 					  &qp->usq.db_rec_data->db_data,  in qedr_create_user_qp()
1955 		qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;  in qedr_create_user_qp()
1956 		qp->rq.max_wr = attrs->cap.max_recv_wr;  in qedr_create_user_qp()
1957 		rc = qedr_db_recovery_add(dev, qp->urq.db_addr,  in qedr_create_user_qp()
1958 					  &qp->urq.db_rec_data->db_data,  in qedr_create_user_qp()
1965 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {  in qedr_create_user_qp()
1966 		qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;  in qedr_create_user_qp()
1971 		qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);  in qedr_create_user_qp()
1972 		qp->urq.db_rec_db2_data.data.value =  in qedr_create_user_qp()
1975 		rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,  in qedr_create_user_qp()
1976 					  &qp->urq.db_rec_db2_data,  in qedr_create_user_qp()
1985 	rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);  in qedr_create_user_qp()
1998 	qp->sq.db = dev->db_addr +  in qedr_set_iwarp_db_info()
2000 	qp->sq.db_data.data.icid = qp->icid;  in qedr_set_iwarp_db_info()
2002 	rc = qedr_db_recovery_add(dev, qp->sq.db,  in qedr_set_iwarp_db_info()
2003 				  &qp->sq.db_data,  in qedr_set_iwarp_db_info()
2009 	qp->rq.db = dev->db_addr +  in qedr_set_iwarp_db_info()
2011 	qp->rq.db_data.data.icid = qp->icid;  in qedr_set_iwarp_db_info()
2012 	qp->rq.iwarp_db2 = dev->db_addr +  in qedr_set_iwarp_db_info()
2014 	qp->rq.iwarp_db2_data.data.icid = qp->icid;  in qedr_set_iwarp_db_info()
2015 	qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;  in qedr_set_iwarp_db_info()
2017 	rc = qedr_db_recovery_add(dev, qp->rq.db,  in qedr_set_iwarp_db_info()
2018 				  &qp->rq.db_data,  in qedr_set_iwarp_db_info()
2024 	rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,  in qedr_set_iwarp_db_info()
2025 				  &qp->rq.iwarp_db2_data,  in qedr_set_iwarp_db_info()
2048 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);  in qedr_roce_create_kernel_qp()
2052 	in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);  in qedr_roce_create_kernel_qp()
2053 	in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);  in qedr_roce_create_kernel_qp()
2059 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);  in qedr_roce_create_kernel_qp()
2063 	in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);  in qedr_roce_create_kernel_qp()
2064 	in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);  in qedr_roce_create_kernel_qp()
2066 	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,  in qedr_roce_create_kernel_qp()
2069 	if (!qp->qed_qp)  in qedr_roce_create_kernel_qp()
2070 		return -EINVAL;  in qedr_roce_create_kernel_qp()
2072 	qp->qp_id = out_params.qp_id;  in qedr_roce_create_kernel_qp()
2073 	qp->icid = out_params.icid;  in qedr_roce_create_kernel_qp()
2091 	in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,  in qedr_iwarp_create_kernel_qp()
2095 	in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,  in qedr_iwarp_create_kernel_qp()
2100 	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,  in qedr_iwarp_create_kernel_qp()
2103 	if (!qp->qed_qp)  in qedr_iwarp_create_kernel_qp()
2104 		return -EINVAL;  in qedr_iwarp_create_kernel_qp()
2114 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);  in qedr_iwarp_create_kernel_qp()
2124 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);  in qedr_iwarp_create_kernel_qp()
2128 	qp->qp_id = out_params.qp_id;  in qedr_iwarp_create_kernel_qp()
2129 	qp->icid = out_params.icid;  in qedr_iwarp_create_kernel_qp()
2134 	dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);  in qedr_iwarp_create_kernel_qp()
2141 	dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);  in qedr_cleanup_kernel()
2142 	kfree(qp->wqe_wr_id);  in qedr_cleanup_kernel()
2144 	dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);  in qedr_cleanup_kernel()
2145 	kfree(qp->rqe_wr_id);  in qedr_cleanup_kernel()
2148 	if (qp->qp_type == IB_QPT_GSI)  in qedr_cleanup_kernel()
2151 	qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);  in qedr_cleanup_kernel()
2153 	if (!qp->srq) {  in qedr_cleanup_kernel()
2154 		qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);  in qedr_cleanup_kernel()
2156 		if (rdma_protocol_iwarp(&dev->ibdev, 1))  in qedr_cleanup_kernel()
2157 			qedr_db_recovery_del(dev, qp->rq.iwarp_db2,  in qedr_cleanup_kernel()
2158 					     &qp->rq.iwarp_db2_data);  in qedr_cleanup_kernel()
2169 	int rc = -EINVAL;  in qedr_create_kernel_qp()
2175 	qp->create_type = QEDR_QP_CREATE_KERNEL;  in qedr_create_kernel_qp()
2187 	qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,  in qedr_create_kernel_qp()
2188 			      dev->attr.max_sqe);  in qedr_create_kernel_qp()
2190 	qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),  in qedr_create_kernel_qp()
2192 	if (!qp->wqe_wr_id) {  in qedr_create_kernel_qp()
2194 		return -ENOMEM;  in qedr_create_kernel_qp()
2205 	qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);  in qedr_create_kernel_qp()
2208 	qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),  in qedr_create_kernel_qp()
2210 	if (!qp->rqe_wr_id) {  in qedr_create_kernel_qp()
2213 		kfree(qp->wqe_wr_id);  in qedr_create_kernel_qp()
2214 		return -ENOMEM;  in qedr_create_kernel_qp()
2219 	n_sq_entries = attrs->cap.max_send_wr;  in qedr_create_kernel_qp()
2220 	n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);  in qedr_create_kernel_qp()
2224 	n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;  in qedr_create_kernel_qp()
2226 	if (rdma_protocol_iwarp(&dev->ibdev, 1))  in qedr_create_kernel_qp()
2246 	if (qp->qp_type != IB_QPT_GSI) {  in qedr_free_qp_resources()
2247 		rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);  in qedr_free_qp_resources()
2252 	if (qp->create_type == QEDR_QP_CREATE_USER)  in qedr_free_qp_resources()
2264 	struct ib_pd *ibpd = ibqp->pd;  in qedr_create_qp()
2266 	struct qedr_dev *dev = get_qedr_dev(ibqp->device);  in qedr_create_qp()
2270 	if (attrs->create_flags)  in qedr_create_qp()
2271 		return -EOPNOTSUPP;  in qedr_create_qp()
2273 	if (attrs->qp_type == IB_QPT_XRC_TGT)  in qedr_create_qp()
2274 		xrcd = get_qedr_xrcd(attrs->xrcd);  in qedr_create_qp()
2287 		 udata ? "user library" : "kernel", attrs->event_handler, pd,  in qedr_create_qp()
2288 		 get_qedr_cq(attrs->send_cq),  in qedr_create_qp()
2289 		 get_qedr_cq(attrs->send_cq)->icid,  in qedr_create_qp()
2290 		 get_qedr_cq(attrs->recv_cq),  in qedr_create_qp()
2291 		 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);  in qedr_create_qp()
2295 	if (attrs->qp_type == IB_QPT_GSI)  in qedr_create_qp()
2306 	qp->ibqp.qp_num = qp->qp_id;  in qedr_create_qp()
2308 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {  in qedr_create_qp()
2309 		rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);  in qedr_create_qp()
2318 	return -EFAULT;  in qedr_create_qp()
2379 			status = -EINVAL;  in qedr_update_qp_state()
2390 			if (rdma_protocol_roce(&dev->ibdev, 1)) {  in qedr_update_qp_state()
2391 				writel(qp->rq.db_data.raw, qp->rq.db);  in qedr_update_qp_state()
2398 			status = -EINVAL;  in qedr_update_qp_state()
2403 		/* RTR->XXX */  in qedr_update_qp_state()
2411 			status = -EINVAL;  in qedr_update_qp_state()
2416 		/* RTS->XXX */  in qedr_update_qp_state()
2424 			status = -EINVAL;  in qedr_update_qp_state()
2429 		/* SQD->XXX */  in qedr_update_qp_state()
2436 			status = -EINVAL;  in qedr_update_qp_state()
2441 		/* ERR->XXX */  in qedr_update_qp_state()
2444 			if ((qp->rq.prod != qp->rq.cons) ||  in qedr_update_qp_state()
2445 			    (qp->sq.prod != qp->sq.cons)) {  in qedr_update_qp_state()
2447 					  "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",  in qedr_update_qp_state()
2448 					  qp->rq.prod, qp->rq.cons, qp->sq.prod,  in qedr_update_qp_state()
2449 					  qp->sq.cons);  in qedr_update_qp_state()
2450 				status = -EINVAL;  in qedr_update_qp_state()
2454 			status = -EINVAL;  in qedr_update_qp_state()
2459 		status = -EINVAL;  in qedr_update_qp_state()
2471 	struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);  in qedr_modify_qp()
2472 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);  in qedr_modify_qp()
2479 		 attr->qp_state);  in qedr_modify_qp()
2482 		return -EOPNOTSUPP;  in qedr_modify_qp()
2484 	old_qp_state = qedr_get_ibqp_state(qp->state);  in qedr_modify_qp()
2486 		new_qp_state = attr->qp_state;  in qedr_modify_qp()
2490 	if (rdma_protocol_roce(&dev->ibdev, 1)) {  in qedr_modify_qp()
2492 					ibqp->qp_type, attr_mask)) {  in qedr_modify_qp()
2496 			       attr_mask, qp->qp_id, ibqp->qp_type,  in qedr_modify_qp()
2498 			rc = -EINVAL;  in qedr_modify_qp()
2507 		qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);  in qedr_modify_qp()
2516 		if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {  in qedr_modify_qp()
2517 			rc = -EINVAL;  in qedr_modify_qp()
2525 		qp->qkey = attr->qkey;  in qedr_modify_qp()
2530 		qp_params.incoming_rdma_read_en = attr->qp_access_flags &  in qedr_modify_qp()
2532 		qp_params.incoming_rdma_write_en = attr->qp_access_flags &  in qedr_modify_qp()
2534 		qp_params.incoming_atomic_en = attr->qp_access_flags &  in qedr_modify_qp()
2539 		if (rdma_protocol_iwarp(&dev->ibdev, 1))  in qedr_modify_qp()
2540 			return -EINVAL;  in qedr_modify_qp()
2543 			if (attr->path_mtu < IB_MTU_256 ||  in qedr_modify_qp()
2544 			    attr->path_mtu > IB_MTU_4096) {  in qedr_modify_qp()
2546 				rc = -EINVAL;  in qedr_modify_qp()
2549 			qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),  in qedr_modify_qp()
2551 							 (dev->ndev->mtu)));  in qedr_modify_qp()
2554 		if (!qp->mtu) {  in qedr_modify_qp()
2555 			qp->mtu =  in qedr_modify_qp()
2556 			ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));  in qedr_modify_qp()
2557 			pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);  in qedr_modify_qp()
2563 		qp_params.traffic_class_tos = grh->traffic_class;  in qedr_modify_qp()
2564 		qp_params.flow_label = grh->flow_label;  in qedr_modify_qp()
2565 		qp_params.hop_limit_ttl = grh->hop_limit;  in qedr_modify_qp()
2567 		qp->sgid_idx = grh->sgid_index;  in qedr_modify_qp()
2573 			       grh->sgid_index, rc);  in qedr_modify_qp()
2577 		rc = qedr_get_dmac(dev, &attr->ah_attr,  in qedr_modify_qp()
2583 		ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);  in qedr_modify_qp()
2594 		qp_params.mtu = qp->mtu;  in qedr_modify_qp()
2600 		if (qp->mtu)  in qedr_modify_qp()
2601 			qp_params.mtu = qp->mtu;  in qedr_modify_qp()
2604 			    ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));  in qedr_modify_qp()
2617 		 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).  in qedr_modify_qp()
2621 		if (attr->timeout)  in qedr_modify_qp()
2623 					1 << max_t(int, attr->timeout - 8, 0);  in qedr_modify_qp()
2627 		qp->timeout = attr->timeout;  in qedr_modify_qp()
2633 		qp_params.retry_cnt = attr->retry_cnt;  in qedr_modify_qp()
2639 		qp_params.rnr_retry_cnt = attr->rnr_retry;  in qedr_modify_qp()
2645 		qp_params.rq_psn = attr->rq_psn;  in qedr_modify_qp()
2646 		qp->rq_psn = attr->rq_psn;  in qedr_modify_qp()
2650 		if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {  in qedr_modify_qp()
2651 			rc = -EINVAL;  in qedr_modify_qp()
2654 			       attr->max_rd_atomic,  in qedr_modify_qp()
2655 			       dev->attr.max_qp_req_rd_atomic_resc);  in qedr_modify_qp()
2661 		qp_params.max_rd_atomic_req = attr->max_rd_atomic;  in qedr_modify_qp()
2667 		qp_params.min_rnr_nak_timer = attr->min_rnr_timer;  in qedr_modify_qp()
2673 		qp_params.sq_psn = attr->sq_psn;  in qedr_modify_qp()
2674 		qp->sq_psn = attr->sq_psn;  in qedr_modify_qp()
2678 		if (attr->max_dest_rd_atomic >  in qedr_modify_qp()
2679 		    dev->attr.max_qp_resp_rd_atomic_resc) {  in qedr_modify_qp()
2682 			       attr->max_dest_rd_atomic,  in qedr_modify_qp()
2683 			       dev->attr.max_qp_resp_rd_atomic_resc);  in qedr_modify_qp()
2685 			rc = -EINVAL;  in qedr_modify_qp()
2691 		qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;  in qedr_modify_qp()
2698 		qp_params.dest_qp = attr->dest_qp_num;  in qedr_modify_qp()
2699 		qp->dest_qp_num = attr->dest_qp_num;  in qedr_modify_qp()
2702 	cur_state = qp->state;  in qedr_modify_qp()
2709 	if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&  in qedr_modify_qp()
2711 		qp->state = QED_ROCE_QP_STATE_ERR;  in qedr_modify_qp()
2713 	if (qp->qp_type != IB_QPT_GSI)  in qedr_modify_qp()
2714 		rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,  in qedr_modify_qp()
2715 					      qp->qed_qp, &qp_params);  in qedr_modify_qp()
2718 		if ((qp->qp_type != IB_QPT_GSI) && (!udata))  in qedr_modify_qp()
2721 		qp->state = qp_params.new_state;  in qedr_modify_qp()
2732 	if (params->incoming_rdma_write_en)  in qedr_to_ib_qp_acc_flags()
2734 	if (params->incoming_rdma_read_en)  in qedr_to_ib_qp_acc_flags()
2736 	if (params->incoming_atomic_en)  in qedr_to_ib_qp_acc_flags()
2748 	struct qedr_dev *dev = qp->dev;  in qedr_query_qp()
2755 	if (qp->qp_type != IB_QPT_GSI) {  in qedr_query_qp()
2756 		rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);  in qedr_query_qp()
2759 		qp_attr->qp_state = qedr_get_ibqp_state(params.state);  in qedr_query_qp()
2761 		qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);  in qedr_query_qp()
2764 	qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);  in qedr_query_qp()
2765 	qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);  in qedr_query_qp()
2766 	qp_attr->path_mig_state = IB_MIG_MIGRATED;  in qedr_query_qp()
2767 	qp_attr->rq_psn = params.rq_psn;  in qedr_query_qp()
2768 	qp_attr->sq_psn = params.sq_psn;  in qedr_query_qp()
2769 	qp_attr->dest_qp_num = params.dest_qp;  in qedr_query_qp()
2771 	qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);  in qedr_query_qp()
2773 	qp_attr->cap.max_send_wr = qp->sq.max_wr;  in qedr_query_qp()
2774 	qp_attr->cap.max_recv_wr = qp->rq.max_wr;  in qedr_query_qp()
2775 	qp_attr->cap.max_send_sge = qp->sq.max_sges;  in qedr_query_qp()
2776 	qp_attr->cap.max_recv_sge = qp->rq.max_sges;  in qedr_query_qp()
2777 	qp_attr->cap.max_inline_data = dev->attr.max_inline;  in qedr_query_qp()
2778 	qp_init_attr->cap = qp_attr->cap;  in qedr_query_qp()
2780 	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;  in qedr_query_qp()
2781 	rdma_ah_set_grh(&qp_attr->ah_attr, NULL,  in qedr_query_qp()
2782 			params.flow_label, qp->sgid_idx,  in qedr_query_qp()
2784 	rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);  in qedr_query_qp()
2785 	rdma_ah_set_port_num(&qp_attr->ah_attr, 1);  in qedr_query_qp()
2786 	rdma_ah_set_sl(&qp_attr->ah_attr, 0);  in qedr_query_qp()
2787 	qp_attr->timeout = qp->timeout;  in qedr_query_qp()
2788 	qp_attr->rnr_retry = params.rnr_retry;  in qedr_query_qp()
2789 	qp_attr->retry_cnt = params.retry_cnt;  in qedr_query_qp()
2790 	qp_attr->min_rnr_timer = params.min_rnr_nak_timer;  in qedr_query_qp()
2791 	qp_attr->pkey_index = params.pkey_index;  in qedr_query_qp()
2792 	qp_attr->port_num = 1;  in qedr_query_qp()
2793 	rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);  in qedr_query_qp()
2794 	rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);  in qedr_query_qp()
2795 	qp_attr->alt_pkey_index = 0;  in qedr_query_qp()
2796 	qp_attr->alt_port_num = 0;  in qedr_query_qp()
2797 	qp_attr->alt_timeout = 0;  in qedr_query_qp()
2798 	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));  in qedr_query_qp()
2800 	qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;  in qedr_query_qp()
2801 	qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;  in qedr_query_qp()
2802 	qp_attr->max_rd_atomic = params.max_rd_atomic;  in qedr_query_qp()
2803 	qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;  in qedr_query_qp()
2806 		 qp_attr->cap.max_inline_data);  in qedr_query_qp()
2815 	struct qedr_dev *dev = qp->dev;  in qedr_destroy_qp()
2820 		 qp, qp->qp_type);  in qedr_destroy_qp()
2822 	if (rdma_protocol_roce(&dev->ibdev, 1)) {  in qedr_destroy_qp()
2823 		if ((qp->state != QED_ROCE_QP_STATE_RESET) &&  in qedr_destroy_qp()
2824 		    (qp->state != QED_ROCE_QP_STATE_ERR) &&  in qedr_destroy_qp()
2825 		    (qp->state != QED_ROCE_QP_STATE_INIT)) {  in qedr_destroy_qp()
2839 				     &qp->iwarp_cm_flags))  in qedr_destroy_qp()
2840 			wait_for_completion(&qp->iwarp_cm_comp);  in qedr_destroy_qp()
2853 				     &qp->iwarp_cm_flags))  in qedr_destroy_qp()
2854 			wait_for_completion(&qp->iwarp_cm_comp);  in qedr_destroy_qp()
2857 	if (qp->qp_type == IB_QPT_GSI)  in qedr_destroy_qp()
2864 	if (rdma_protocol_iwarp(&dev->ibdev, 1))  in qedr_destroy_qp()
2865 		xa_erase(&dev->qps, qp->qp_id);  in qedr_destroy_qp()
2869 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {  in qedr_destroy_qp()
2870 		qedr_iw_qp_rem_ref(&qp->ibqp);  in qedr_destroy_qp()
2871 		wait_for_completion(&qp->qp_rel_comp);  in qedr_destroy_qp()
2882 	rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);  in qedr_create_ah()
2891 	rdma_destroy_ah_attr(&ah->attr);  in qedr_destroy_ah()
2899 	if (info->pbl_table)  in free_mr_info()
2900 		list_add_tail(&info->pbl_table->list_entry,  in free_mr_info()
2901 			      &info->free_pbl_list);  in free_mr_info()
2903 	if (!list_empty(&info->inuse_pbl_list))  in free_mr_info()
2904 		list_splice(&info->inuse_pbl_list, &info->free_pbl_list);  in free_mr_info()
2906 	list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {  in free_mr_info()
2907 		list_del(&pbl->list_entry);  in free_mr_info()
2908 		qedr_free_pbl(dev, &info->pbl_info, pbl);  in free_mr_info()
2918 	INIT_LIST_HEAD(&info->free_pbl_list);  in init_mr_info()
2919 	INIT_LIST_HEAD(&info->inuse_pbl_list);  in init_mr_info()
2921 	rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,  in init_mr_info()
2926 	info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);  in init_mr_info()
2927 	if (IS_ERR(info->pbl_table)) {  in init_mr_info()
2928 		rc = PTR_ERR(info->pbl_table);  in init_mr_info()
2933 		 &info->pbl_table->pa);  in init_mr_info()
2938 	tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);  in init_mr_info()
2944 	list_add_tail(&tmp->list_entry, &info->free_pbl_list);  in init_mr_info()
2946 	DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);  in init_mr_info()
2958 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);  in qedr_reg_user_mr()
2961 	int rc = -ENOMEM;  in qedr_reg_user_mr()
2966 		 pd->pd_id, start, len, usr_addr, acc);  in qedr_reg_user_mr()
2969 		return ERR_PTR(-EINVAL);  in qedr_reg_user_mr()
2975 	mr->type = QEDR_MR_USER;  in qedr_reg_user_mr()
2977 	mr->umem = ib_umem_get(ibpd->device, start, len, acc);  in qedr_reg_user_mr()
2978 	if (IS_ERR(mr->umem)) {  in qedr_reg_user_mr()
2979 		rc = -EFAULT;  in qedr_reg_user_mr()
2983 	rc = init_mr_info(dev, &mr->info,  in qedr_reg_user_mr()
2984 			  ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);  in qedr_reg_user_mr()
2988 	qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,  in qedr_reg_user_mr()
2989 			   &mr->info.pbl_info, PAGE_SHIFT);  in qedr_reg_user_mr()
2991 	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);  in qedr_reg_user_mr()
2993 		if (rc == -EINVAL)  in qedr_reg_user_mr()
3002 	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;  in qedr_reg_user_mr()
3003 	mr->hw_mr.key = 0;  in qedr_reg_user_mr()
3004 	mr->hw_mr.pd = pd->pd_id;  in qedr_reg_user_mr()
3005 	mr->hw_mr.local_read = 1;  in qedr_reg_user_mr()
3006 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;  in qedr_reg_user_mr()
3007 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;  in qedr_reg_user_mr()
3008 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;  in qedr_reg_user_mr()
3009 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;  in qedr_reg_user_mr()
3010 	mr->hw_mr.mw_bind = false;  in qedr_reg_user_mr()
3011 	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;  in qedr_reg_user_mr()
3012 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;  in qedr_reg_user_mr()
3013 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);  in qedr_reg_user_mr()
3014 	mr->hw_mr.page_size_log = PAGE_SHIFT;  in qedr_reg_user_mr()
3015 	mr->hw_mr.length = len;  in qedr_reg_user_mr()
3016 	mr->hw_mr.vaddr = usr_addr;  in qedr_reg_user_mr()
3017 	mr->hw_mr.phy_mr = false;  in qedr_reg_user_mr()
3018 	mr->hw_mr.dma_mr = false;  in qedr_reg_user_mr()
3020 	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);  in qedr_reg_user_mr()
3026 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;  in qedr_reg_user_mr()
3027 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||  in qedr_reg_user_mr()
3028 	    mr->hw_mr.remote_atomic)  in qedr_reg_user_mr()
3029 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;  in qedr_reg_user_mr()
3032 		 mr->ibmr.lkey);  in qedr_reg_user_mr()
3033 	return &mr->ibmr;  in qedr_reg_user_mr()
3036 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);  in qedr_reg_user_mr()
3038 	qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);  in qedr_reg_user_mr()
3047 	struct qedr_dev *dev = get_qedr_dev(ib_mr->device);  in qedr_dereg_mr()
3050 	rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);  in qedr_dereg_mr()
3054 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);  in qedr_dereg_mr()
3056 	if (mr->type != QEDR_MR_DMA)  in qedr_dereg_mr()
3057 		free_mr_info(dev, &mr->info);  in qedr_dereg_mr()
3060 	ib_umem_release(mr->umem);  in qedr_dereg_mr()
3071 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);  in __qedr_alloc_mr()
3073 	int rc = -ENOMEM;  in __qedr_alloc_mr()
3076 		 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,  in __qedr_alloc_mr()
3083 	mr->dev = dev;  in __qedr_alloc_mr()
3084 	mr->type = QEDR_MR_FRMR;  in __qedr_alloc_mr()
3086 	rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);  in __qedr_alloc_mr()
3090 	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);  in __qedr_alloc_mr()
3092 		if (rc == -EINVAL)  in __qedr_alloc_mr()
3101 	mr->hw_mr.tid_type = QED_RDMA_TID_FMR;  in __qedr_alloc_mr()
3102 	mr->hw_mr.key = 0;  in __qedr_alloc_mr()
3103 	mr->hw_mr.pd = pd->pd_id;  in __qedr_alloc_mr()
3104 	mr->hw_mr.local_read = 1;  in __qedr_alloc_mr()
3105 	mr->hw_mr.local_write = 0;  in __qedr_alloc_mr()
3106 	mr->hw_mr.remote_read = 0;  in __qedr_alloc_mr()
3107 	mr->hw_mr.remote_write = 0;  in __qedr_alloc_mr()
3108 	mr->hw_mr.remote_atomic = 0;  in __qedr_alloc_mr()
3109 	mr->hw_mr.mw_bind = false;  in __qedr_alloc_mr()
3110 	mr->hw_mr.pbl_ptr = 0;  in __qedr_alloc_mr()
3111 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;  in __qedr_alloc_mr()
3112 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);  in __qedr_alloc_mr()
3113 	mr->hw_mr.length = 0;  in __qedr_alloc_mr()
3114 	mr->hw_mr.vaddr = 0;  in __qedr_alloc_mr()
3115 	mr->hw_mr.phy_mr = true;  in __qedr_alloc_mr()
3116 	mr->hw_mr.dma_mr = false;  in __qedr_alloc_mr()
3118 	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);  in __qedr_alloc_mr()
3124 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;  in __qedr_alloc_mr()
3125 	mr->ibmr.rkey = mr->ibmr.lkey;  in __qedr_alloc_mr()
3127 	DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);  in __qedr_alloc_mr()
3131 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);  in __qedr_alloc_mr()
3133 	qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);  in __qedr_alloc_mr()
3145 		return ERR_PTR(-EINVAL);  in qedr_alloc_mr()
3150 		return ERR_PTR(-EINVAL);  in qedr_alloc_mr()
3152 	return &mr->ibmr;  in qedr_alloc_mr()
3162 	if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {  in qedr_set_page()
3163 		DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);  in qedr_set_page()
3164 		return -ENOMEM;  in qedr_set_page()
3167 	DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",  in qedr_set_page()
3168 		 mr->npages, addr);  in qedr_set_page()
3170 	pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);  in qedr_set_page()
3171 	pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);  in qedr_set_page()
3172 	pbe = (struct regpair *)pbl_table->va;  in qedr_set_page()
3173 	pbe +=  mr->npages % pbes_in_page;  in qedr_set_page()
3174 	pbe->lo = cpu_to_le32((u32)addr);  in qedr_set_page()
3175 	pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));  in qedr_set_page()
3177 	mr->npages++;  in qedr_set_page()
3184 	int work = info->completed - info->completed_handled - 1;  in handle_completed_mrs()
3187 	while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {  in handle_completed_mrs()
3195 		pbl = list_first_entry(&info->inuse_pbl_list,  in handle_completed_mrs()
3197 		list_move_tail(&pbl->list_entry, &info->free_pbl_list);  in handle_completed_mrs()
3198 		info->completed_handled++;  in handle_completed_mrs()
3207 	mr->npages = 0;  in qedr_map_mr_sg()
3209 	handle_completed_mrs(mr->dev, &mr->info);  in qedr_map_mr_sg()
3215 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);  in qedr_get_dma_mr()
3222 		return ERR_PTR(-ENOMEM);  in qedr_get_dma_mr()
3224 	mr->type = QEDR_MR_DMA;  in qedr_get_dma_mr()
3226 	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);  in qedr_get_dma_mr()
3228 		if (rc == -EINVAL)  in qedr_get_dma_mr()
3237 	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;  in qedr_get_dma_mr()
3238 	mr->hw_mr.pd = pd->pd_id;  in qedr_get_dma_mr()
3239 	mr->hw_mr.local_read = 1;  in qedr_get_dma_mr()
3240 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;  in qedr_get_dma_mr()
3241 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;  in qedr_get_dma_mr()
3242 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;  in qedr_get_dma_mr()
3243 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;  in qedr_get_dma_mr()
3244 	mr->hw_mr.dma_mr = true;  in qedr_get_dma_mr()
3246 	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);  in qedr_get_dma_mr()
3252 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;  in qedr_get_dma_mr()
3253 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||  in qedr_get_dma_mr()
3254 	    mr->hw_mr.remote_atomic)  in qedr_get_dma_mr()
3255 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;  in qedr_get_dma_mr()
3257 	DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);  in qedr_get_dma_mr()
3258 	return &mr->ibmr;  in qedr_get_dma_mr()
3261 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);  in qedr_get_dma_mr()
3269 	return (((wq->prod + 1) % wq->max_wr) == wq->cons);  in qedr_wq_is_full()
3296 	u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);  in qedr_prepare_sq_inline_data()
3316 	for (i = 0; i < wr->num_sge; i++) {  in qedr_prepare_sq_inline_data()
3317 		u32 len = wr->sg_list[i].length;  in qedr_prepare_sq_inline_data()
3318 		void *src = (void *)(uintptr_t)wr->sg_list[i].addr;  in qedr_prepare_sq_inline_data()
3325 				wqe = (char *)qed_chain_produce(&qp->sq.pbl);  in qedr_prepare_sq_inline_data()
3337 			seg_siz -= cur;  in qedr_prepare_sq_inline_data()
3341 			len -= cur;  in qedr_prepare_sq_inline_data()
3343 			/* Swap fully-completed segments */  in qedr_prepare_sq_inline_data()
3358 		DMA_REGPAIR_LE(sge->addr, vaddr);		\
3359 		(sge)->length = cpu_to_le32(vlength);		\
3360 		(sge)->flags = cpu_to_le32(vflags);		\
3365 		DMA_REGPAIR_LE(hdr->wr_id, vwr_id);		\
3366 		(hdr)->num_sges = num_sge;			\
3371 		DMA_REGPAIR_LE(sge->addr, vaddr);		\
3372 		(sge)->length = cpu_to_le32(vlength);		\
3373 		(sge)->l_key = cpu_to_le32(vlkey);		\
3382 	for (i = 0; i < wr->num_sge; i++) {  in qedr_prepare_sq_sges()
3383 		struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);  in qedr_prepare_sq_sges()
3385 		DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);  in qedr_prepare_sq_sges()
3386 		sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);  in qedr_prepare_sq_sges()
3387 		sge->length = cpu_to_le32(wr->sg_list[i].length);  in qedr_prepare_sq_sges()
3388 		data_size += wr->sg_list[i].length;  in qedr_prepare_sq_sges()
3392 		*wqe_size += wr->num_sge;  in qedr_prepare_sq_sges()
3404 	rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);  in qedr_prepare_sq_rdma_data()
3405 	DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);  in qedr_prepare_sq_rdma_data()
3407 	if (wr->send_flags & IB_SEND_INLINE &&  in qedr_prepare_sq_rdma_data()
3408 	    (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||  in qedr_prepare_sq_rdma_data()
3409 	     wr->opcode == IB_WR_RDMA_WRITE)) {  in qedr_prepare_sq_rdma_data()
3413 		return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,  in qedr_prepare_sq_rdma_data()
3414 						   bad_wr, &rwqe->flags, flags);  in qedr_prepare_sq_rdma_data()
3417 	return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);  in qedr_prepare_sq_rdma_data()
3428 	if (wr->send_flags & IB_SEND_INLINE) {  in qedr_prepare_sq_send_data()
3432 		return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,  in qedr_prepare_sq_send_data()
3433 						   bad_wr, &swqe->flags, flags);  in qedr_prepare_sq_send_data()
3436 	return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);  in qedr_prepare_sq_send_data()
3443 	struct qedr_mr *mr = get_qedr_mr(wr->mr);  in qedr_prepare_reg()
3446 	fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);  in qedr_prepare_reg()
3447 	fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);  in qedr_prepare_reg()
3448 	fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);  in qedr_prepare_reg()
3449 	fwqe1->l_key = wr->key;  in qedr_prepare_reg()
3451 	fwqe2->access_ctrl = 0;  in qedr_prepare_reg()
3453 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,  in qedr_prepare_reg()
3454 		   !!(wr->access & IB_ACCESS_REMOTE_READ));  in qedr_prepare_reg()
3455 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,  in qedr_prepare_reg()
3456 		   !!(wr->access & IB_ACCESS_REMOTE_WRITE));  in qedr_prepare_reg()
3457 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,  in qedr_prepare_reg()
3458 		   !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));  in qedr_prepare_reg()
3459 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);  in qedr_prepare_reg()
3460 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,  in qedr_prepare_reg()
3461 		   !!(wr->access & IB_ACCESS_LOCAL_WRITE));  in qedr_prepare_reg()
3462 	fwqe2->fmr_ctrl = 0;  in qedr_prepare_reg()
3464 	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,  in qedr_prepare_reg()
3465 		   ilog2(mr->ibmr.page_size) - 12);  in qedr_prepare_reg()
3467 	fwqe2->length_hi = 0;  in qedr_prepare_reg()
3468 	fwqe2->length_lo = mr->ibmr.length;  in qedr_prepare_reg()
3469 	fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);  in qedr_prepare_reg()
3470 	fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);  in qedr_prepare_reg()
3472 	qp->wqe_wr_id[qp->sq.prod].mr = mr;  in qedr_prepare_reg()
3507 	struct qedr_dev *dev = qp->dev;  in qedr_can_post_send()
3510 	err_wr = wr->num_sge > qp->sq.max_sges;  in qedr_can_post_send()
3511 	wq_is_full = qedr_wq_is_full(&qp->sq);  in qedr_can_post_send()
3512 	pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <  in qedr_can_post_send()
3515 		if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {  in qedr_can_post_send()
3519 			qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;  in qedr_can_post_send()
3522 		if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {  in qedr_can_post_send()
3526 			qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;  in qedr_can_post_send()
3530 		    !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {  in qedr_can_post_send()
3534 			qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;  in qedr_can_post_send()
3544 	struct qedr_dev *dev = get_qedr_dev(ibqp->device);  in __qedr_post_send()
3562 		return -ENOMEM;  in __qedr_post_send()
3565 	wqe = qed_chain_produce(&qp->sq.pbl);  in __qedr_post_send()
3566 	qp->wqe_wr_id[qp->sq.prod].signaled =  in __qedr_post_send()
3567 		!!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;  in __qedr_post_send()
3569 	wqe->flags = 0;  in __qedr_post_send()
3570 	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,  in __qedr_post_send()
3571 		   !!(wr->send_flags & IB_SEND_SOLICITED));  in __qedr_post_send()
3572 	comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;  in __qedr_post_send()
3573 	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);  in __qedr_post_send()
3574 	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,  in __qedr_post_send()
3575 		   !!(wr->send_flags & IB_SEND_FENCE));  in __qedr_post_send()
3576 	wqe->prev_wqe_size = qp->prev_wqe_size;  in __qedr_post_send()
3578 	qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);  in __qedr_post_send()
3580 	switch (wr->opcode) {  in __qedr_post_send()
3582 		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {  in __qedr_post_send()
3583 			rc = -EINVAL;  in __qedr_post_send()
3587 		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;  in __qedr_post_send()
3589 		swqe->wqe_size = 2;  in __qedr_post_send()
3590 		swqe2 = qed_chain_produce(&qp->sq.pbl);  in __qedr_post_send()
3592 		swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));  in __qedr_post_send()
3595 		swqe->length = cpu_to_le32(length);  in __qedr_post_send()
3596 		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;  in __qedr_post_send()
3597 		qp->prev_wqe_size = swqe->wqe_size;  in __qedr_post_send()
3598 		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;  in __qedr_post_send()
3601 		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;  in __qedr_post_send()
3604 		swqe->wqe_size = 2;  in __qedr_post_send()
3605 		swqe2 = qed_chain_produce(&qp->sq.pbl);  in __qedr_post_send()
3608 		swqe->length = cpu_to_le32(length);  in __qedr_post_send()
3609 		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;  in __qedr_post_send()
3610 		qp->prev_wqe_size = swqe->wqe_size;  in __qedr_post_send()
3611 		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;  in __qedr_post_send()
3614 		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;  in __qedr_post_send()
3616 		swqe2 = qed_chain_produce(&qp->sq.pbl);  in __qedr_post_send()
3617 		swqe->wqe_size = 2;  in __qedr_post_send()
3618 		swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);  in __qedr_post_send()
3621 		swqe->length = cpu_to_le32(length);  in __qedr_post_send()
3622 		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;  in __qedr_post_send()
3623 		qp->prev_wqe_size = swqe->wqe_size;  in __qedr_post_send()
3624 		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;  in __qedr_post_send()
3628 		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {  in __qedr_post_send()
3629 			rc = -EINVAL;  in __qedr_post_send()
3633 		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;  in __qedr_post_send()
3636 		rwqe->wqe_size = 2;  in __qedr_post_send()
3637 		rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));  in __qedr_post_send()
3638 		rwqe2 = qed_chain_produce(&qp->sq.pbl);  in __qedr_post_send()
3641 		rwqe->length = cpu_to_le32(length);  in __qedr_post_send()
3642 		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;  in __qedr_post_send()
3643 		qp->prev_wqe_size = rwqe->wqe_size;  in __qedr_post_send()
3644 		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;  in __qedr_post_send()
3647 		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;  in __qedr_post_send()
3650 		rwqe->wqe_size = 2;  in __qedr_post_send()
3651 		rwqe2 = qed_chain_produce(&qp->sq.pbl);  in __qedr_post_send()
3654 		rwqe->length = cpu_to_le32(length);  in __qedr_post_send()
3655 		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;  in __qedr_post_send()
3656 		qp->prev_wqe_size = rwqe->wqe_size;  in __qedr_post_send()
3657 		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;  in __qedr_post_send()
3660 		SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);  in __qedr_post_send()
3664 		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;  in __qedr_post_send()
3667 		rwqe->wqe_size = 2;  in __qedr_post_send()
3668 		rwqe2 = qed_chain_produce(&qp->sq.pbl);  in __qedr_post_send()
3671 		rwqe->length = cpu_to_le32(length);  in __qedr_post_send()
3672 		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;  in __qedr_post_send()
3673 		qp->prev_wqe_size = rwqe->wqe_size;  in __qedr_post_send()
3674 		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;  in __qedr_post_send()
3680 		awqe1->wqe_size = 4;  in __qedr_post_send()
3682 		awqe2 = qed_chain_produce(&qp->sq.pbl);  in __qedr_post_send()
3683 		DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);  in __qedr_post_send()
3684 		awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);  in __qedr_post_send()
3686 		awqe3 = qed_chain_produce(&qp->sq.pbl);  in __qedr_post_send()
3688 		if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {  in __qedr_post_send()
3689 			wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;  in __qedr_post_send()
3690 			DMA_REGPAIR_LE(awqe3->swap_data,  in __qedr_post_send()
3691 				       atomic_wr(wr)->compare_add);  in __qedr_post_send()
3693 			wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;  in __qedr_post_send()
3694 			DMA_REGPAIR_LE(awqe3->swap_data,  in __qedr_post_send()
3695 				       atomic_wr(wr)->swap);  in __qedr_post_send()
3696 			DMA_REGPAIR_LE(awqe3->cmp_data,  in __qedr_post_send()
3697 				       atomic_wr(wr)->compare_add);  in __qedr_post_send()
3702 		qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;  in __qedr_post_send()
3703 		qp->prev_wqe_size = awqe1->wqe_size;  in __qedr_post_send()
3708 		iwqe->wqe_size = 1;  in __qedr_post_send()
3710 		iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;  in __qedr_post_send()
3711 		iwqe->inv_l_key = wr->ex.invalidate_rkey;  in __qedr_post_send()
3712 		qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;  in __qedr_post_send()
3713 		qp->prev_wqe_size = iwqe->wqe_size;  in __qedr_post_send()
3717 		wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;  in __qedr_post_send()
3719 		fwqe1->wqe_size = 2;  in __qedr_post_send()
3728 		qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;  in __qedr_post_send()
3729 		qp->prev_wqe_size = fwqe1->wqe_size;  in __qedr_post_send()
3732 		DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);  in __qedr_post_send()
3733 		rc = -EINVAL;  in __qedr_post_send()
3744 		value = le16_to_cpu(qp->sq.db_data.data.value);  in __qedr_post_send()
3745 		qed_chain_set_prod(&qp->sq.pbl, value, wqe);  in __qedr_post_send()
3748 		qp->prev_wqe_size = wqe->prev_wqe_size;  in __qedr_post_send()
3749 		rc = -EINVAL;  in __qedr_post_send()
3759 	struct qedr_dev *dev = get_qedr_dev(ibqp->device);  in qedr_post_send()
3766 	if (qp->qp_type == IB_QPT_GSI)  in qedr_post_send()
3769 	spin_lock_irqsave(&qp->q_lock, flags);  in qedr_post_send()
3771 	if (rdma_protocol_roce(&dev->ibdev, 1)) {  in qedr_post_send()
3772 		if ((qp->state != QED_ROCE_QP_STATE_RTS) &&  in qedr_post_send()
3773 		    (qp->state != QED_ROCE_QP_STATE_ERR) &&  in qedr_post_send()
3774 		    (qp->state != QED_ROCE_QP_STATE_SQD)) {  in qedr_post_send()
3775 			spin_unlock_irqrestore(&qp->q_lock, flags);  in qedr_post_send()
3778 				 "QP in wrong state! QP icid=0x%x state %d\n",  in qedr_post_send()
3779 				 qp->icid, qp->state);  in qedr_post_send()
3780 			return -EINVAL;  in qedr_post_send()
3789 		qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;  in qedr_post_send()
3791 		qedr_inc_sw_prod(&qp->sq);  in qedr_post_send()
3793 		qp->sq.db_data.data.value++;  in qedr_post_send()
3795 		wr = wr->next;  in qedr_post_send()
3804 	 * qp->wqe_wr_id is accessed during qedr_poll_cq, as  in qedr_post_send()
3812 	writel(qp->sq.db_data.raw, qp->sq.db);  in qedr_post_send()
3814 	spin_unlock_irqrestore(&qp->q_lock, flags);  in qedr_post_send()
3827 	used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);  in qedr_srq_elem_left()
3829 	return hw_srq->max_wr - used;  in qedr_srq_elem_left()
3837 	struct qedr_dev *dev = srq->dev;  in qedr_post_srq_recv()
3843 	spin_lock_irqsave(&srq->lock, flags);  in qedr_post_srq_recv()
3845 	hw_srq = &srq->hw_srq;  in qedr_post_srq_recv()
3846 	pbl = &srq->hw_srq.pbl;  in qedr_post_srq_recv()
3852 		    wr->num_sge > srq->hw_srq.max_sges) {  in qedr_post_srq_recv()
3854 			       hw_srq->wr_prod_cnt,  in qedr_post_srq_recv()
3855 			       atomic_read(&hw_srq->wr_cons_cnt),  in qedr_post_srq_recv()
3856 			       wr->num_sge, srq->hw_srq.max_sges);  in qedr_post_srq_recv()
3857 			status = -ENOMEM;  in qedr_post_srq_recv()
3863 		num_sge = wr->num_sge;  in qedr_post_srq_recv()
3865 		SRQ_HDR_SET(hdr, wr->wr_id, num_sge);  in qedr_post_srq_recv()
3867 		srq->hw_srq.wr_prod_cnt++;  in qedr_post_srq_recv()
3868 		hw_srq->wqe_prod++;  in qedr_post_srq_recv()
3869 		hw_srq->sge_prod++;  in qedr_post_srq_recv()
3873 			 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);  in qedr_post_srq_recv()
3875 		for (i = 0; i < wr->num_sge; i++) {  in qedr_post_srq_recv()
3879 			SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,  in qedr_post_srq_recv()
3880 				    wr->sg_list[i].length, wr->sg_list[i].lkey);  in qedr_post_srq_recv()
3884 				 i, srq_sge->length, srq_sge->l_key,  in qedr_post_srq_recv()
3885 				 srq_sge->addr.hi, srq_sge->addr.lo);  in qedr_post_srq_recv()
3886 			hw_srq->sge_prod++;  in qedr_post_srq_recv()
3898 		srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);  in qedr_post_srq_recv()
3901 		srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);  in qedr_post_srq_recv()
3903 		wr = wr->next;  in qedr_post_srq_recv()
3906 	DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",  in qedr_post_srq_recv()
3908 	spin_unlock_irqrestore(&srq->lock, flags);  in qedr_post_srq_recv()
3917 	struct qedr_dev *dev = qp->dev;  in qedr_post_recv()
3921 	if (qp->qp_type == IB_QPT_GSI)  in qedr_post_recv()
3924 	spin_lock_irqsave(&qp->q_lock, flags);  in qedr_post_recv()
3929 		if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <  in qedr_post_recv()
3931 		    wr->num_sge > qp->rq.max_sges) {  in qedr_post_recv()
3933 			       qed_chain_get_elem_left_u32(&qp->rq.pbl),  in qedr_post_recv()
3934 			       QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,  in qedr_post_recv()
3935 			       qp->rq.max_sges);  in qedr_post_recv()
3936 			status = -ENOMEM;  in qedr_post_recv()
3940 		for (i = 0; i < wr->num_sge; i++) {  in qedr_post_recv()
3943 			    qed_chain_produce(&qp->rq.pbl);  in qedr_post_recv()
3950 					  wr->num_sge);  in qedr_post_recv()
3953 				  wr->sg_list[i].lkey);  in qedr_post_recv()
3955 			RQ_SGE_SET(rqe, wr->sg_list[i].addr,  in qedr_post_recv()
3956 				   wr->sg_list[i].length, flags);  in qedr_post_recv()
3959 		/* Special case of no sges. FW requires between 1-4 sges...  in qedr_post_recv()
3963 		if (!wr->num_sge) {  in qedr_post_recv()
3966 			    qed_chain_produce(&qp->rq.pbl);  in qedr_post_recv()
3978 		qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;  in qedr_post_recv()
3979 		qp->rqe_wr_id[qp->rq.prod].wqe_size = i;  in qedr_post_recv()
3981 		qedr_inc_sw_prod(&qp->rq);  in qedr_post_recv()
3983 		/* qp->rqe_wr_id is accessed during qedr_poll_cq, as  in qedr_post_recv()
3992 		qp->rq.db_data.data.value++;  in qedr_post_recv()
3994 		writel(qp->rq.db_data.raw, qp->rq.db);  in qedr_post_recv()
3996 		if (rdma_protocol_iwarp(&dev->ibdev, 1)) {  in qedr_post_recv()
3997 			writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);  in qedr_post_recv()
4000 		wr = wr->next;  in qedr_post_recv()
4003 	spin_unlock_irqrestore(&qp->q_lock, flags);  in qedr_post_recv()
4010 	struct rdma_cqe_requester *resp_cqe = &cqe->req;  in is_valid_cqe()
4012 	return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==  in is_valid_cqe()
4013 		cq->pbl_toggle;  in is_valid_cqe()
4018 	struct rdma_cqe_requester *resp_cqe = &cqe->req;  in cqe_get_qp()
4021 	qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,  in cqe_get_qp()
4022 						   resp_cqe->qp_handle.lo,  in cqe_get_qp()
4029 	struct rdma_cqe_requester *resp_cqe = &cqe->req;  in cqe_get_type()
4031 	return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);  in cqe_get_type()
4037 	return cq->latest_cqe;  in get_cqe()
4049 	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)  in qedr_chk_if_fmr()
4050 		qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;  in qedr_chk_if_fmr()
4060 	while (num_entries && qp->sq.wqe_cons != hw_cons) {  in process_req()
4061 		if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {  in process_req()
4068 		wc->status = status;  in process_req()
4069 		wc->vendor_err = 0;  in process_req()
4070 		wc->wc_flags = 0;  in process_req()
4071 		wc->src_qp = qp->id;  in process_req()
4072 		wc->qp = &qp->ibqp;  in process_req()
4074 		wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;  in process_req()
4075 		wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;  in process_req()
4077 		switch (wc->opcode) {  in process_req()
4079 			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;  in process_req()
4083 			wc->byte_len = 8;  in process_req()
4086 			qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;  in process_req()
4090 			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;  in process_req()
4096 		num_entries--;  in process_req()
4100 		while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)  in process_req()
4101 			qed_chain_consume(&qp->sq.pbl);  in process_req()
4102 		qedr_inc_sw_cons(&qp->sq);  in process_req()
4115 	switch (req->status) {  in qedr_poll_cq_req()
4117 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,  in qedr_poll_cq_req()
4121 		if (qp->state != QED_ROCE_QP_STATE_ERR)  in qedr_poll_cq_req()
4123 				 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4124 				 cq->icid, qp->icid);  in qedr_poll_cq_req()
4125 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,  in qedr_poll_cq_req()
4130 		qp->state = QED_ROCE_QP_STATE_ERR;  in qedr_poll_cq_req()
4132 				  req->sq_cons - 1, IB_WC_SUCCESS, 0);  in qedr_poll_cq_req()
4138 			switch (req->status) {  in qedr_poll_cq_req()
4141 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4142 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4147 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4148 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4153 …       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4154 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4159 …		       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4160 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4165 …     "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4166 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4171 …   "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4172 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4177 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4178 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4183 …		       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4184 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4189 …	       "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4190 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4195 …      "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4196 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4201 				       "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",  in qedr_poll_cq_req()
4202 				       cq->icid, qp->icid);  in qedr_poll_cq_req()
4205 			cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,  in qedr_poll_cq_req()
4238 	wc->status = IB_WC_SUCCESS;  in qedr_set_ok_cqe_resp_wc()
4239 	wc->byte_len = le32_to_cpu(resp->length);  in qedr_set_ok_cqe_resp_wc()
4241 	if (resp->flags & QEDR_RESP_IMM) {  in qedr_set_ok_cqe_resp_wc()
4242 		wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));  in qedr_set_ok_cqe_resp_wc()
4243 		wc->wc_flags |= IB_WC_WITH_IMM;  in qedr_set_ok_cqe_resp_wc()
4245 		if (resp->flags & QEDR_RESP_RDMA)  in qedr_set_ok_cqe_resp_wc()
4246 			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;  in qedr_set_ok_cqe_resp_wc()
4248 		if (resp->flags & QEDR_RESP_INV)  in qedr_set_ok_cqe_resp_wc()
4249 			return -EINVAL;  in qedr_set_ok_cqe_resp_wc()
4251 	} else if (resp->flags & QEDR_RESP_INV) {  in qedr_set_ok_cqe_resp_wc()
4252 		wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);  in qedr_set_ok_cqe_resp_wc()
4253 		wc->wc_flags |= IB_WC_WITH_INVALIDATE;  in qedr_set_ok_cqe_resp_wc()
4255 		if (resp->flags & QEDR_RESP_RDMA)  in qedr_set_ok_cqe_resp_wc()
4256 			return -EINVAL;  in qedr_set_ok_cqe_resp_wc()
4258 	} else if (resp->flags & QEDR_RESP_RDMA) {  in qedr_set_ok_cqe_resp_wc()
4259 		return -EINVAL;  in qedr_set_ok_cqe_resp_wc()
4270 	wc->opcode = IB_WC_RECV;  in __process_resp_one()
4271 	wc->wc_flags = 0;  in __process_resp_one()
4273 	if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {  in __process_resp_one()
4276 			       "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",  in __process_resp_one()
4277 			       cq, cq->icid, resp->flags);  in __process_resp_one()
4280 		wc->status = qedr_cqe_resp_status_to_ib(resp->status);  in __process_resp_one()
4281 		if (wc->status == IB_WC_GENERAL_ERR)  in __process_resp_one()
4283 			       "CQ %p (icid=%d) contains an invalid CQE status %d\n",  in __process_resp_one()
4284 			       cq, cq->icid, resp->status);  in __process_resp_one()
4288 	wc->vendor_err = 0;  in __process_resp_one()
4289 	wc->src_qp = qp->id;  in __process_resp_one()
4290 	wc->qp = &qp->ibqp;  in __process_resp_one()
4291 	wc->wr_id = wr_id;  in __process_resp_one()
4298 	struct qedr_srq *srq = qp->srq;  in process_resp_one_srq()
4301 	wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),  in process_resp_one_srq()
4302 			 le32_to_cpu(resp->srq_wr_id.lo), u64);  in process_resp_one_srq()
4304 	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {  in process_resp_one_srq()
4305 		wc->status = IB_WC_WR_FLUSH_ERR;  in process_resp_one_srq()
4306 		wc->vendor_err = 0;  in process_resp_one_srq()
4307 		wc->wr_id = wr_id;  in process_resp_one_srq()
4308 		wc->byte_len = 0;  in process_resp_one_srq()
4309 		wc->src_qp = qp->id;  in process_resp_one_srq()
4310 		wc->qp = &qp->ibqp;  in process_resp_one_srq()
4311 		wc->wr_id = wr_id;  in process_resp_one_srq()
4315 	atomic_inc(&srq->hw_srq.wr_cons_cnt);  in process_resp_one_srq()
4323 	u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;  in process_resp_one()
4327 	while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)  in process_resp_one()
4328 		qed_chain_consume(&qp->rq.pbl);  in process_resp_one()
4329 	qedr_inc_sw_cons(&qp->rq);  in process_resp_one()
4339 	while (num_entries && qp->rq.wqe_cons != hw_cons) {  in process_resp_flush()
4341 		wc->status = IB_WC_WR_FLUSH_ERR;  in process_resp_flush()
4342 		wc->vendor_err = 0;  in process_resp_flush()
4343 		wc->wc_flags = 0;  in process_resp_flush()
4344 		wc->src_qp = qp->id;  in process_resp_flush()
4345 		wc->byte_len = 0;  in process_resp_flush()
4346 		wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;  in process_resp_flush()
4347 		wc->qp = &qp->ibqp;  in process_resp_flush()
4348 		num_entries--;  in process_resp_flush()
4351 		while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)  in process_resp_flush()
4352 			qed_chain_consume(&qp->rq.pbl);  in process_resp_flush()
4353 		qedr_inc_sw_cons(&qp->rq);  in process_resp_flush()
4362 	if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {  in try_consume_resp_cqe()
4388 	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {  in qedr_poll_cq_resp()
4390 					 resp->rq_cons_or_srq_id);  in qedr_poll_cq_resp()
4404 	if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {  in try_consume_req_cqe()
4412 	struct qedr_dev *dev = get_qedr_dev(ibcq->device);  in qedr_poll_cq()
4420 	if (cq->destroyed) {  in qedr_poll_cq()
4422 		       "warning: poll was invoked after destroy for cq %p (icid=%d)\n",  in qedr_poll_cq()
4423 		       cq, cq->icid);  in qedr_poll_cq()
4427 	if (cq->cq_type == QEDR_CQ_TYPE_GSI)  in qedr_poll_cq()
4430 	spin_lock_irqsave(&cq->cq_lock, flags);  in qedr_poll_cq()
4431 	cqe = cq->latest_cqe;  in qedr_poll_cq()
4432 	old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);  in qedr_poll_cq()
4446 		wc->qp = &qp->ibqp;  in qedr_poll_cq()
4451 					       &cqe->req);  in qedr_poll_cq()
4452 			try_consume_req_cqe(cq, qp, &cqe->req, &update);  in qedr_poll_cq()
4456 						&cqe->resp, &update);  in qedr_poll_cq()
4460 						    wc, &cqe->resp);  in qedr_poll_cq()
4468 		num_entries -= cnt;  in qedr_poll_cq()
4474 	new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);  in qedr_poll_cq()
4476 	cq->cq_cons += new_cons - old_cons;  in qedr_poll_cq()
4482 		doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);  in qedr_poll_cq()
4484 	spin_unlock_irqrestore(&cq->cq_lock, flags);  in qedr_poll_cq()