1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
4 /* Copyright (c) 2008-2019, IBM Corporation */
9 #include <linux/dma-mapping.h>
32 	if (xa_alloc_cyclic(&sdev->mem_xa, &id, m, limit, &next,  in siw_mem_add()
34 		return -ENOMEM;  in siw_mem_add()
37 	m->stag = id << 8;  in siw_mem_add()
56 	mem = xa_load(&sdev->mem_xa, stag_index);  in siw_mem_id2obj()
57 	if (likely(mem && kref_get_unless_zero(&mem->ref))) {  in siw_mem_id2obj()
68 	int i, num_pages = umem->num_pages;  in siw_umem_release()
70 	if (umem->base_mem)  in siw_umem_release()
71 		ib_umem_release(umem->base_mem);  in siw_umem_release()
74 		kfree(umem->page_chunk[i].plist);  in siw_umem_release()
75 		num_pages -= PAGES_PER_CHUNK;  in siw_umem_release()
77 	kfree(umem->page_chunk);  in siw_umem_release()
84 	struct siw_device *sdev = to_siw_dev(pd->device);  in siw_mr_add_mem()
90 		return -ENOMEM;  in siw_mr_add_mem()
92 	mem->mem_obj = mem_obj;  in siw_mr_add_mem()
93 	mem->stag_valid = 0;  in siw_mr_add_mem()
94 	mem->sdev = sdev;  in siw_mr_add_mem()
95 	mem->va = start;  in siw_mr_add_mem()
96 	mem->len = len;  in siw_mr_add_mem()
97 	mem->pd = pd;  in siw_mr_add_mem()
98 	mem->perms = rights & IWARP_ACCESS_MASK;  in siw_mr_add_mem()
99 	kref_init(&mem->ref);  in siw_mr_add_mem()
104 	if (xa_alloc_cyclic(&sdev->mem_xa, &id, mem, limit, &next,  in siw_mr_add_mem()
107 		return -ENOMEM;  in siw_mr_add_mem()
110 	mr->mem = mem;  in siw_mr_add_mem()
112 	mem->stag = id << 8;  in siw_mr_add_mem()
113 	mr->base_mr.lkey = mr->base_mr.rkey = mem->stag;  in siw_mr_add_mem()
120 	struct siw_mem *mem = mr->mem, *found;  in siw_mr_drop_mem()
122 	mem->stag_valid = 0;  in siw_mr_drop_mem()
127 	found = xa_erase(&mem->sdev->mem_xa, mem->stag >> 8);  in siw_mr_drop_mem()
136 	siw_dbg_mem(mem, "free mem, pbl: %s\n", mem->is_pbl ? "y" : "n");  in siw_free_mem()
138 	if (!mem->is_mw && mem->mem_obj) {  in siw_free_mem()
139 		if (mem->is_pbl == 0)  in siw_free_mem()
140 			siw_umem_release(mem->umem);  in siw_free_mem()
142 			kfree(mem->pbl);  in siw_free_mem()
163 	if (!mem->stag_valid) {  in siw_check_mem()
164 		siw_dbg_pd(pd, "STag 0x%08x invalid\n", mem->stag);  in siw_check_mem()
165 		return -E_STAG_INVALID;  in siw_check_mem()
167 	if (mem->pd != pd) {  in siw_check_mem()
168 		siw_dbg_pd(pd, "STag 0x%08x: PD mismatch\n", mem->stag);  in siw_check_mem()
169 		return -E_PD_MISMATCH;  in siw_check_mem()
174 	if ((mem->perms & perms) < perms) {  in siw_check_mem()
176 			   mem->perms, perms);  in siw_check_mem()
177 		return -E_ACCESS_PERM;  in siw_check_mem()
182 	if (addr < mem->va || addr + len > mem->va + mem->len) {  in siw_check_mem()
184 		siw_dbg_pd(pd, "[0x%pK, 0x%pK] out of bounds\n",  in siw_check_mem()
187 		siw_dbg_pd(pd, "[0x%pK, 0x%pK] STag=0x%08x\n",  in siw_check_mem()
188 			   (void *)(uintptr_t)mem->va,  in siw_check_mem()
189 			   (void *)(uintptr_t)(mem->va + mem->len),  in siw_check_mem()
190 			   mem->stag);  in siw_check_mem()
192 		return -E_BASE_BOUNDS;  in siw_check_mem()
209  * NOTE: Function references SGE's memory object (mem->obj)
211  * released if check failed. If mem->obj is already valid, no new
217 	struct siw_device *sdev = to_siw_dev(pd->device);  in siw_check_sge()
221 	if (len + off > sge->length) {  in siw_check_sge()
222 		rv = -E_BASE_BOUNDS;  in siw_check_sge()
226 		new = siw_mem_id2obj(sdev, sge->lkey >> 8);  in siw_check_sge()
228 			siw_dbg_pd(pd, "STag unknown: 0x%08x\n", sge->lkey);  in siw_check_sge()
229 			rv = -E_STAG_INVALID;  in siw_check_sge()
234 	/* Check if user re-registered with different STag key */  in siw_check_sge()
235 	if (unlikely((*mem)->stag != sge->lkey)) {  in siw_check_sge()
236 		siw_dbg_mem((*mem), "STag mismatch: 0x%08x\n", sge->lkey);  in siw_check_sge()
237 		rv = -E_STAG_INVALID;  in siw_check_sge()
240 	rv = siw_check_mem(pd, *mem, sge->laddr + off, perms, len);  in siw_check_sge()
263 		if (!(wqe->sqe.flags & SIW_WQE_INLINE))  in siw_wqe_put_mem()
264 			siw_unref_mem_sgl(wqe->mem, wqe->sqe.num_sge);  in siw_wqe_put_mem()
268 		siw_unref_mem_sgl(wqe->mem, wqe->rqe.num_sge);  in siw_wqe_put_mem()
272 		siw_unref_mem_sgl(wqe->mem, 1);  in siw_wqe_put_mem()
286 	struct siw_device *sdev = to_siw_dev(pd->device);  in siw_invalidate_stag()
292 		return -EINVAL;  in siw_invalidate_stag()
294 	if (unlikely(mem->pd != pd)) {  in siw_invalidate_stag()
296 		rv = -EACCES;  in siw_invalidate_stag()
303 	mem->stag_valid = 0;  in siw_invalidate_stag()
321 	while (i < pbl->num_buf) {  in siw_pbl_get_buffer()
322 		struct siw_pble *pble = &pbl->pbe[i];  in siw_pbl_get_buffer()
324 		if (pble->pbl_off + pble->size > off) {  in siw_pbl_get_buffer()
325 			u64 pble_off = off - pble->pbl_off;  in siw_pbl_get_buffer()
328 				*len = pble->size - pble_off;  in siw_pbl_get_buffer()
332 			return pble->addr + pble_off;  in siw_pbl_get_buffer()
346 		return ERR_PTR(-EINVAL);  in siw_pbl_alloc()
350 		return ERR_PTR(-ENOMEM);  in siw_pbl_alloc()
352 	pbl->max_buf = num_buf;  in siw_pbl_alloc()
368 		return ERR_PTR(-EINVAL);  in siw_umem_get()
371 	num_pages = PAGE_ALIGN(start + len - first_page_va) >> PAGE_SHIFT;  in siw_umem_get()
376 		return ERR_PTR(-ENOMEM);  in siw_umem_get()
378 	umem->page_chunk =  in siw_umem_get()
380 	if (!umem->page_chunk) {  in siw_umem_get()
381 		rv = -ENOMEM;  in siw_umem_get()
390 	umem->fp_addr = first_page_va;  in siw_umem_get()
391 	umem->base_mem = base_mem;  in siw_umem_get()
393 	sgt = &base_mem->sgt_append.sgt;  in siw_umem_get()
394 	__sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0);  in siw_umem_get()
397 		rv = -EINVAL;  in siw_umem_get()
406 			rv = -ENOMEM;  in siw_umem_get()
409 		umem->page_chunk[i].plist = plist;  in siw_umem_get()
410 		while (nents--) {  in siw_umem_get()
412 			umem->num_pages++;  in siw_umem_get()
413 			num_pages--;  in siw_umem_get()