Lines Matching +full:unit +full:- +full:address
15 * - Redistributions of source code must retain the above
19 * - Redistributions in binary form must reproduce the above
54 hop_num = hr_dev->caps.qpc_hop_num; in hns_roce_check_whether_mhop()
57 hop_num = hr_dev->caps.mpt_hop_num; in hns_roce_check_whether_mhop()
60 hop_num = hr_dev->caps.cqc_hop_num; in hns_roce_check_whether_mhop()
63 hop_num = hr_dev->caps.srqc_hop_num; in hns_roce_check_whether_mhop()
66 hop_num = hr_dev->caps.sccc_hop_num; in hns_roce_check_whether_mhop()
69 hop_num = hr_dev->caps.qpc_timer_hop_num; in hns_roce_check_whether_mhop()
72 hop_num = hr_dev->caps.cqc_timer_hop_num; in hns_roce_check_whether_mhop()
75 hop_num = hr_dev->caps.gmv_hop_num; in hns_roce_check_whether_mhop()
126 struct device *dev = hr_dev->dev; in get_hem_table_config()
130 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz in get_hem_table_config()
132 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz in get_hem_table_config()
134 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; in get_hem_table_config()
135 mhop->hop_num = hr_dev->caps.qpc_hop_num; in get_hem_table_config()
138 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz in get_hem_table_config()
140 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz in get_hem_table_config()
142 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; in get_hem_table_config()
143 mhop->hop_num = hr_dev->caps.mpt_hop_num; in get_hem_table_config()
146 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz in get_hem_table_config()
148 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz in get_hem_table_config()
150 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; in get_hem_table_config()
151 mhop->hop_num = hr_dev->caps.cqc_hop_num; in get_hem_table_config()
154 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz in get_hem_table_config()
156 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz in get_hem_table_config()
158 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; in get_hem_table_config()
159 mhop->hop_num = hr_dev->caps.sccc_hop_num; in get_hem_table_config()
162 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz in get_hem_table_config()
164 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz in get_hem_table_config()
166 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; in get_hem_table_config()
167 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; in get_hem_table_config()
170 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz in get_hem_table_config()
172 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz in get_hem_table_config()
174 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; in get_hem_table_config()
175 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; in get_hem_table_config()
178 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz in get_hem_table_config()
180 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz in get_hem_table_config()
182 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; in get_hem_table_config()
183 mhop->hop_num = hr_dev->caps.srqc_hop_num; in get_hem_table_config()
186 mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + in get_hem_table_config()
188 mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz + in get_hem_table_config()
190 mhop->ba_l0_num = hr_dev->caps.gmv_bt_num; in get_hem_table_config()
191 mhop->hop_num = hr_dev->caps.gmv_hop_num; in get_hem_table_config()
194 dev_err(dev, "table %u not support multi-hop addressing!\n", in get_hem_table_config()
196 return -EINVAL; in get_hem_table_config()
206 struct device *dev = hr_dev->dev; in hns_roce_calc_hem_mhop()
212 if (get_hem_table_config(hr_dev, mhop, table->type)) in hns_roce_calc_hem_mhop()
213 return -EINVAL; in hns_roce_calc_hem_mhop()
222 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); in hns_roce_calc_hem_mhop()
223 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; in hns_roce_calc_hem_mhop()
224 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : in hns_roce_calc_hem_mhop()
225 mhop->bt_chunk_size; in hns_roce_calc_hem_mhop()
226 table_idx = *obj / (chunk_size / table->obj_size); in hns_roce_calc_hem_mhop()
229 mhop->l2_idx = table_idx & (chunk_ba_num - 1); in hns_roce_calc_hem_mhop()
230 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); in hns_roce_calc_hem_mhop()
231 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; in hns_roce_calc_hem_mhop()
234 mhop->l1_idx = table_idx & (chunk_ba_num - 1); in hns_roce_calc_hem_mhop()
235 mhop->l0_idx = table_idx / chunk_ba_num; in hns_roce_calc_hem_mhop()
238 mhop->l0_idx = table_idx; in hns_roce_calc_hem_mhop()
242 table->type, mhop->hop_num); in hns_roce_calc_hem_mhop()
243 return -EINVAL; in hns_roce_calc_hem_mhop()
245 if (mhop->l0_idx >= mhop->ba_l0_num) in hns_roce_calc_hem_mhop()
246 mhop->l0_idx %= mhop->ba_l0_num; in hns_roce_calc_hem_mhop()
263 dev_err(hr_dev->dev, "invalid hem_alloc_size: %lu!\n", in hns_roce_alloc_hem()
273 buf = dma_alloc_coherent(hr_dev->dev, hem_alloc_size, in hns_roce_alloc_hem()
274 &hem->dma, gfp_mask); in hns_roce_alloc_hem()
278 hem->buf = buf; in hns_roce_alloc_hem()
279 hem->size = hem_alloc_size; in hns_roce_alloc_hem()
293 dma_free_coherent(hr_dev->dev, hem->size, hem->buf, hem->dma); in hns_roce_free_hem()
303 struct device *dev = hr_dev->dev; in calc_hem_config()
314 l0_idx = mhop->l0_idx; in calc_hem_config()
315 l1_idx = mhop->l1_idx; in calc_hem_config()
316 l2_idx = mhop->l2_idx; in calc_hem_config()
317 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; in calc_hem_config()
318 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); in calc_hem_config()
321 index->l1 = l0_idx * chunk_ba_num + l1_idx; in calc_hem_config()
322 index->l0 = l0_idx; in calc_hem_config()
323 index->buf = l0_idx * chunk_ba_num * chunk_ba_num + in calc_hem_config()
327 index->l0 = l0_idx; in calc_hem_config()
328 index->buf = l0_idx * chunk_ba_num + l1_idx; in calc_hem_config()
331 index->buf = l0_idx; in calc_hem_config()
335 table->type, mhop->hop_num); in calc_hem_config()
336 return -EINVAL; in calc_hem_config()
339 if (unlikely(index->buf >= table->num_hem)) { in calc_hem_config()
341 table->type, index->buf, table->num_hem); in calc_hem_config()
342 return -EINVAL; in calc_hem_config()
353 u32 bt_size = mhop->bt_chunk_size; in free_mhop_hem()
354 struct device *dev = hr_dev->dev; in free_mhop_hem()
356 if (index->inited & HEM_INDEX_BUF) { in free_mhop_hem()
357 hns_roce_free_hem(hr_dev, table->hem[index->buf]); in free_mhop_hem()
358 table->hem[index->buf] = NULL; in free_mhop_hem()
361 if (index->inited & HEM_INDEX_L1) { in free_mhop_hem()
362 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], in free_mhop_hem()
363 table->bt_l1_dma_addr[index->l1]); in free_mhop_hem()
364 table->bt_l1[index->l1] = NULL; in free_mhop_hem()
367 if (index->inited & HEM_INDEX_L0) { in free_mhop_hem()
368 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], in free_mhop_hem()
369 table->bt_l0_dma_addr[index->l0]); in free_mhop_hem()
370 table->bt_l0[index->l0] = NULL; in free_mhop_hem()
379 u32 bt_size = mhop->bt_chunk_size; in alloc_mhop_hem()
380 struct device *dev = hr_dev->dev; in alloc_mhop_hem()
387 if ((check_whether_bt_num_3(table->type, mhop->hop_num) || in alloc_mhop_hem()
388 check_whether_bt_num_2(table->type, mhop->hop_num)) && in alloc_mhop_hem()
389 !table->bt_l0[index->l0]) { in alloc_mhop_hem()
390 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, in alloc_mhop_hem()
391 &table->bt_l0_dma_addr[index->l0], in alloc_mhop_hem()
393 if (!table->bt_l0[index->l0]) { in alloc_mhop_hem()
394 ret = -ENOMEM; in alloc_mhop_hem()
397 index->inited |= HEM_INDEX_L0; in alloc_mhop_hem()
401 if (check_whether_bt_num_3(table->type, mhop->hop_num) && in alloc_mhop_hem()
402 !table->bt_l1[index->l1]) { in alloc_mhop_hem()
403 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, in alloc_mhop_hem()
404 &table->bt_l1_dma_addr[index->l1], in alloc_mhop_hem()
406 if (!table->bt_l1[index->l1]) { in alloc_mhop_hem()
407 ret = -ENOMEM; in alloc_mhop_hem()
410 index->inited |= HEM_INDEX_L1; in alloc_mhop_hem()
411 *(table->bt_l0[index->l0] + mhop->l1_idx) = in alloc_mhop_hem()
412 table->bt_l1_dma_addr[index->l1]; in alloc_mhop_hem()
419 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; in alloc_mhop_hem()
421 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size, flag); in alloc_mhop_hem()
422 if (!table->hem[index->buf]) { in alloc_mhop_hem()
423 ret = -ENOMEM; in alloc_mhop_hem()
427 index->inited |= HEM_INDEX_BUF; in alloc_mhop_hem()
428 bt_ba = table->hem[index->buf]->dma; in alloc_mhop_hem()
430 if (table->type < HEM_TYPE_MTT) { in alloc_mhop_hem()
431 if (mhop->hop_num == 2) in alloc_mhop_hem()
432 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; in alloc_mhop_hem()
433 else if (mhop->hop_num == 1) in alloc_mhop_hem()
434 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; in alloc_mhop_hem()
435 } else if (mhop->hop_num == 2) { in alloc_mhop_hem()
436 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; in alloc_mhop_hem()
451 struct device *dev = hr_dev->dev; in set_mhop_hem()
455 if (index->inited & HEM_INDEX_L0) { in set_mhop_hem()
456 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); in set_mhop_hem()
463 if (index->inited & HEM_INDEX_L1) { in set_mhop_hem()
464 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); in set_mhop_hem()
471 if (index->inited & HEM_INDEX_BUF) { in set_mhop_hem()
472 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0) in set_mhop_hem()
475 step_idx = mhop->hop_num; in set_mhop_hem()
476 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); in set_mhop_hem()
490 struct device *dev = hr_dev->dev; in hns_roce_table_mhop_get()
499 mutex_lock(&table->mutex); in hns_roce_table_mhop_get()
500 if (table->hem[index.buf]) { in hns_roce_table_mhop_get()
501 refcount_inc(&table->hem[index.buf]->refcount); in hns_roce_table_mhop_get()
511 /* set HEM base address to hardware */ in hns_roce_table_mhop_get()
512 if (table->type < HEM_TYPE_MTT) { in hns_roce_table_mhop_get()
515 dev_err(dev, "set HEM address to HW failed!\n"); in hns_roce_table_mhop_get()
520 refcount_set(&table->hem[index.buf]->refcount, 1); in hns_roce_table_mhop_get()
526 mutex_unlock(&table->mutex); in hns_roce_table_mhop_get()
533 struct device *dev = hr_dev->dev; in hns_roce_table_get()
537 if (hns_roce_check_whether_mhop(hr_dev, table->type)) in hns_roce_table_get()
540 i = obj / (table->table_chunk_size / table->obj_size); in hns_roce_table_get()
542 mutex_lock(&table->mutex); in hns_roce_table_get()
544 if (table->hem[i]) { in hns_roce_table_get()
545 refcount_inc(&table->hem[i]->refcount); in hns_roce_table_get()
549 table->hem[i] = hns_roce_alloc_hem(hr_dev, in hns_roce_table_get()
550 table->table_chunk_size, in hns_roce_table_get()
552 if (!table->hem[i]) { in hns_roce_table_get()
553 ret = -ENOMEM; in hns_roce_table_get()
557 /* Set HEM base address(128K/page, pa) to Hardware */ in hns_roce_table_get()
558 ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); in hns_roce_table_get()
560 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_get()
561 table->hem[i] = NULL; in hns_roce_table_get()
562 dev_err(dev, "set HEM base address to HW failed, ret = %d.\n", in hns_roce_table_get()
567 refcount_set(&table->hem[i]->refcount, 1); in hns_roce_table_get()
569 mutex_unlock(&table->mutex); in hns_roce_table_get()
578 struct device *dev = hr_dev->dev; in clear_mhop_hem()
579 u32 hop_num = mhop->hop_num; in clear_mhop_hem()
584 index->inited = HEM_INDEX_BUF; in clear_mhop_hem()
585 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; in clear_mhop_hem()
586 if (check_whether_bt_num_2(table->type, hop_num)) { in clear_mhop_hem()
587 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
588 chunk_ba_num, table->num_hem)) in clear_mhop_hem()
589 index->inited |= HEM_INDEX_L0; in clear_mhop_hem()
590 } else if (check_whether_bt_num_3(table->type, hop_num)) { in clear_mhop_hem()
591 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
592 chunk_ba_num, table->num_hem)) { in clear_mhop_hem()
593 index->inited |= HEM_INDEX_L1; in clear_mhop_hem()
594 if (hns_roce_check_bt_null(table->bt_l1, index->l1, in clear_mhop_hem()
596 index->inited |= HEM_INDEX_L0; in clear_mhop_hem()
600 if (table->type < HEM_TYPE_MTT) { in clear_mhop_hem()
606 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx); in clear_mhop_hem()
611 if (index->inited & HEM_INDEX_L1) { in clear_mhop_hem()
612 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 1); in clear_mhop_hem()
618 if (index->inited & HEM_INDEX_L0) { in clear_mhop_hem()
619 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0); in clear_mhop_hem()
634 struct device *dev = hr_dev->dev; in hns_roce_table_mhop_put()
644 mutex_lock(&table->mutex); in hns_roce_table_mhop_put()
645 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, in hns_roce_table_mhop_put()
646 &table->mutex)) in hns_roce_table_mhop_put()
652 mutex_unlock(&table->mutex); in hns_roce_table_mhop_put()
658 struct device *dev = hr_dev->dev; in hns_roce_table_put()
662 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_put()
667 i = obj / (table->table_chunk_size / table->obj_size); in hns_roce_table_put()
669 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, in hns_roce_table_put()
670 &table->mutex)) in hns_roce_table_put()
673 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); in hns_roce_table_put()
675 dev_warn_ratelimited(dev, "failed to clear HEM base address, ret = %d.\n", in hns_roce_table_put()
678 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_put()
679 table->hem[i] = NULL; in hns_roce_table_put()
681 mutex_unlock(&table->mutex); in hns_roce_table_put()
698 mutex_lock(&table->mutex); in hns_roce_table_find()
700 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_find()
701 obj_per_chunk = table->table_chunk_size / table->obj_size; in hns_roce_table_find()
702 hem = table->hem[obj / obj_per_chunk]; in hns_roce_table_find()
704 dma_offset = offset = idx_offset * table->obj_size; in hns_roce_table_find()
719 hem = table->hem[hem_idx]; in hns_roce_table_find()
728 *dma_handle = hem->dma + dma_offset; in hns_roce_table_find()
729 addr = hem->buf + offset; in hns_roce_table_find()
732 mutex_unlock(&table->mutex); in hns_roce_table_find()
744 table->table_chunk_size = hr_dev->caps.chunk_sz; in hns_roce_init_hem_table()
745 obj_per_chunk = table->table_chunk_size / obj_size; in hns_roce_init_hem_table()
748 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); in hns_roce_init_hem_table()
749 if (!table->hem) in hns_roce_init_hem_table()
750 return -ENOMEM; in hns_roce_init_hem_table()
760 return -EINVAL; in hns_roce_init_hem_table()
774 table->hem = kcalloc(num_hem, sizeof(*table->hem), in hns_roce_init_hem_table()
776 if (!table->hem) in hns_roce_init_hem_table()
783 table->bt_l1 = kcalloc(num_bt_l1, in hns_roce_init_hem_table()
784 sizeof(*table->bt_l1), in hns_roce_init_hem_table()
786 if (!table->bt_l1) in hns_roce_init_hem_table()
789 table->bt_l1_dma_addr = kcalloc(num_bt_l1, in hns_roce_init_hem_table()
790 sizeof(*table->bt_l1_dma_addr), in hns_roce_init_hem_table()
793 if (!table->bt_l1_dma_addr) in hns_roce_init_hem_table()
799 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), in hns_roce_init_hem_table()
801 if (!table->bt_l0) in hns_roce_init_hem_table()
804 table->bt_l0_dma_addr = kcalloc(num_bt_l0, in hns_roce_init_hem_table()
805 sizeof(*table->bt_l0_dma_addr), in hns_roce_init_hem_table()
807 if (!table->bt_l0_dma_addr) in hns_roce_init_hem_table()
812 table->type = type; in hns_roce_init_hem_table()
813 table->num_hem = num_hem; in hns_roce_init_hem_table()
814 table->obj_size = obj_size; in hns_roce_init_hem_table()
815 mutex_init(&table->mutex); in hns_roce_init_hem_table()
820 kfree(table->bt_l0); in hns_roce_init_hem_table()
821 table->bt_l0 = NULL; in hns_roce_init_hem_table()
824 kfree(table->bt_l1_dma_addr); in hns_roce_init_hem_table()
825 table->bt_l1_dma_addr = NULL; in hns_roce_init_hem_table()
828 kfree(table->bt_l1); in hns_roce_init_hem_table()
829 table->bt_l1 = NULL; in hns_roce_init_hem_table()
832 kfree(table->hem); in hns_roce_init_hem_table()
833 table->hem = NULL; in hns_roce_init_hem_table()
836 return -ENOMEM; in hns_roce_init_hem_table()
849 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : in hns_roce_cleanup_mhop_hem_table()
852 for (i = 0; i < table->num_hem; ++i) { in hns_roce_cleanup_mhop_hem_table()
853 obj = i * buf_chunk_size / table->obj_size; in hns_roce_cleanup_mhop_hem_table()
854 if (table->hem[i]) in hns_roce_cleanup_mhop_hem_table()
858 kfree(table->hem); in hns_roce_cleanup_mhop_hem_table()
859 table->hem = NULL; in hns_roce_cleanup_mhop_hem_table()
860 kfree(table->bt_l1); in hns_roce_cleanup_mhop_hem_table()
861 table->bt_l1 = NULL; in hns_roce_cleanup_mhop_hem_table()
862 kfree(table->bt_l1_dma_addr); in hns_roce_cleanup_mhop_hem_table()
863 table->bt_l1_dma_addr = NULL; in hns_roce_cleanup_mhop_hem_table()
864 kfree(table->bt_l0); in hns_roce_cleanup_mhop_hem_table()
865 table->bt_l0 = NULL; in hns_roce_cleanup_mhop_hem_table()
866 kfree(table->bt_l0_dma_addr); in hns_roce_cleanup_mhop_hem_table()
867 table->bt_l0_dma_addr = NULL; in hns_roce_cleanup_mhop_hem_table()
873 struct device *dev = hr_dev->dev; in hns_roce_cleanup_hem_table()
878 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_cleanup_hem_table()
880 mutex_destroy(&table->mutex); in hns_roce_cleanup_hem_table()
884 for (i = 0; i < table->num_hem; ++i) in hns_roce_cleanup_hem_table()
885 if (table->hem[i]) { in hns_roce_cleanup_hem_table()
886 obj = i * table->table_chunk_size / table->obj_size; in hns_roce_cleanup_hem_table()
887 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0); in hns_roce_cleanup_hem_table()
889 dev_err(dev, "clear HEM base address failed, ret = %d.\n", in hns_roce_cleanup_hem_table()
892 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_cleanup_hem_table()
895 mutex_destroy(&table->mutex); in hns_roce_cleanup_hem_table()
896 kfree(table->hem); in hns_roce_cleanup_hem_table()
901 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) in hns_roce_cleanup_hem()
903 &hr_dev->srq_table.table); in hns_roce_cleanup_hem()
904 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); in hns_roce_cleanup_hem()
905 if (hr_dev->caps.qpc_timer_entry_sz) in hns_roce_cleanup_hem()
907 &hr_dev->qpc_timer_table); in hns_roce_cleanup_hem()
908 if (hr_dev->caps.cqc_timer_entry_sz) in hns_roce_cleanup_hem()
910 &hr_dev->cqc_timer_table); in hns_roce_cleanup_hem()
911 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) in hns_roce_cleanup_hem()
913 &hr_dev->qp_table.sccc_table); in hns_roce_cleanup_hem()
914 if (hr_dev->caps.trrl_entry_sz) in hns_roce_cleanup_hem()
916 &hr_dev->qp_table.trrl_table); in hns_roce_cleanup_hem()
918 if (hr_dev->caps.gmv_entry_sz) in hns_roce_cleanup_hem()
919 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table); in hns_roce_cleanup_hem()
921 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); in hns_roce_cleanup_hem()
922 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); in hns_roce_cleanup_hem()
923 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); in hns_roce_cleanup_hem()
955 hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN, in hem_list_alloc_item()
956 &hem->dma_addr, GFP_KERNEL); in hem_list_alloc_item()
957 if (!hem->addr) { in hem_list_alloc_item()
963 hem->exist_bt = exist_bt; in hem_list_alloc_item()
964 hem->count = count; in hem_list_alloc_item()
965 hem->start = start; in hem_list_alloc_item()
966 hem->end = end; in hem_list_alloc_item()
967 INIT_LIST_HEAD(&hem->list); in hem_list_alloc_item()
968 INIT_LIST_HEAD(&hem->sibling); in hem_list_alloc_item()
976 if (hem->exist_bt) in hem_list_free_item()
977 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, in hem_list_free_item()
978 hem->addr, hem->dma_addr); in hem_list_free_item()
988 list_del(&hem->list); in hem_list_free_all()
998 /* assign L0 table address to hem from root bt */
1002 hem->addr = cpu_addr; in hem_list_assign_bt()
1003 hem->dma_addr = (dma_addr_t)phy_addr; in hem_list_assign_bt()
1009 return (hem->start <= offset && offset <= hem->end); in hem_list_page_is_in_range()
1031 * hopnum base address table levels in hem_list_is_bottom_bt()
1033 * 1 L0 -> buf in hem_list_is_bottom_bt()
1034 * 2 L0 -> L1 -> buf in hem_list_is_bottom_bt()
1035 * 3 L0 -> L1 -> L2 -> buf in hem_list_is_bottom_bt()
1037 return bt_level >= (hopnum ? hopnum - 1 : hopnum); in hem_list_is_bottom_bt()
1041 * calc base address entries num
1043 * @bt_level: base address table level
1044 * @unit: ba entries per bt page
1046 static u64 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) in hem_list_calc_ba_range() argument
1056 * 1 0 unit in hem_list_calc_ba_range()
1057 * ------------ in hem_list_calc_ba_range()
1058 * 2 0 unit * unit in hem_list_calc_ba_range()
1059 * 2 1 unit in hem_list_calc_ba_range()
1060 * ------------ in hem_list_calc_ba_range()
1061 * 3 0 unit * unit * unit in hem_list_calc_ba_range()
1062 * 3 1 unit * unit in hem_list_calc_ba_range()
1063 * 3 2 unit in hem_list_calc_ba_range()
1066 max = hopnum - bt_level; in hem_list_calc_ba_range()
1068 step = step * unit; in hem_list_calc_ba_range()
1077 * @unit: ba entries per bt page
1080 int region_cnt, int unit) in hns_roce_hem_list_calc_root_ba() argument
1089 /* when r->hopnum = 0, the region should not occupy root_ba. */ in hns_roce_hem_list_calc_root_ba()
1090 if (!r->hopnum) in hns_roce_hem_list_calc_root_ba()
1093 if (r->hopnum > 1) { in hns_roce_hem_list_calc_root_ba()
1094 step = hem_list_calc_ba_range(r->hopnum, 1, unit); in hns_roce_hem_list_calc_root_ba()
1096 total += (r->count + step - 1) / step; in hns_roce_hem_list_calc_root_ba()
1098 total += r->count; in hns_roce_hem_list_calc_root_ba()
1106 const struct hns_roce_buf_region *r, int unit, in hem_list_alloc_mid_bt() argument
1113 const int hopnum = r->hopnum; in hem_list_alloc_mid_bt()
1126 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); in hem_list_alloc_mid_bt()
1127 return -EINVAL; in hem_list_alloc_mid_bt()
1130 if (offset < r->offset) { in hem_list_alloc_mid_bt()
1131 dev_err(hr_dev->dev, "invalid offset %d, min %u!\n", in hem_list_alloc_mid_bt()
1132 offset, r->offset); in hem_list_alloc_mid_bt()
1133 return -EINVAL; in hem_list_alloc_mid_bt()
1136 distance = offset - r->offset; in hem_list_alloc_mid_bt()
1137 max_ofs = r->offset + r->count - 1; in hem_list_alloc_mid_bt()
1151 step = hem_list_calc_ba_range(hopnum, level, unit); in hem_list_alloc_mid_bt()
1153 ret = -EINVAL; in hem_list_alloc_mid_bt()
1157 start_aligned = (distance / step) * step + r->offset; in hem_list_alloc_mid_bt()
1158 end = min_t(u64, start_aligned + step - 1, max_ofs); in hem_list_alloc_mid_bt()
1159 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, in hem_list_alloc_mid_bt()
1162 ret = -ENOMEM; in hem_list_alloc_mid_bt()
1166 list_add(&cur->list, &temp_list[level]); in hem_list_alloc_mid_bt()
1168 list_add(&cur->sibling, &temp_list[0]); in hem_list_alloc_mid_bt()
1172 pre = hem_ptrs[level - 1]; in hem_list_alloc_mid_bt()
1173 step = (cur->start - pre->start) / step * BA_BYTE_LEN; in hem_list_alloc_mid_bt()
1174 hem_list_link_bt(pre->addr + step, cur->dma_addr); in hem_list_alloc_mid_bt()
1192 alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num, in alloc_root_hem() argument
1200 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit); in alloc_root_hem()
1202 return ERR_PTR(-ENOMEM); in alloc_root_hem()
1204 if (ba_num > unit) in alloc_root_hem()
1205 return ERR_PTR(-ENOBUFS); in alloc_root_hem()
1209 r = ®ions[region_cnt - 1]; in alloc_root_hem()
1210 hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, in alloc_root_hem()
1213 return ERR_PTR(-ENOMEM); in alloc_root_hem()
1227 /* This is on the has_mtt branch, if r->hopnum in alloc_fake_root_bt()
1232 hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1, in alloc_fake_root_bt()
1233 r->count, !r->hopnum); in alloc_fake_root_bt()
1235 return -ENOMEM; in alloc_fake_root_bt()
1237 /* The root_ba can be reused only when r->hopnum > 0. */ in alloc_fake_root_bt()
1238 if (r->hopnum) in alloc_fake_root_bt()
1240 list_add(&hem->list, branch_head); in alloc_fake_root_bt()
1241 list_add(&hem->sibling, leaf_head); in alloc_fake_root_bt()
1243 /* If r->hopnum == 0, 0 is returned, in alloc_fake_root_bt()
1246 return r->hopnum ? r->count : 0; in alloc_fake_root_bt()
1250 int unit, const struct hns_roce_buf_region *r, in setup_middle_bt() argument
1258 step = hem_list_calc_ba_range(r->hopnum, 1, unit); in setup_middle_bt()
1260 return -EINVAL; in setup_middle_bt()
1264 offset = (hem->start - r->offset) / step * BA_BYTE_LEN; in setup_middle_bt()
1265 hem_list_link_bt(cpu_base + offset, hem->dma_addr); in setup_middle_bt()
1274 int unit, int max_ba_num, struct hns_roce_hem_head *head, in setup_root_hem() argument
1284 root_hem = list_first_entry(&head->root, in setup_root_hem()
1287 return -ENOMEM; in setup_root_hem()
1292 if (!r->count) in setup_root_hem()
1296 cpu_base = root_hem->addr + total * BA_BYTE_LEN; in setup_root_hem()
1297 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; in setup_root_hem()
1300 * which's address share to all regions. in setup_root_hem()
1302 if (hem_list_is_bottom_bt(r->hopnum, 0)) in setup_root_hem()
1304 &head->branch[i], &head->leaf); in setup_root_hem()
1306 ret = setup_middle_bt(hr_dev, cpu_base, unit, r, in setup_root_hem()
1307 &hem_list->mid_bt[i][1]); in setup_root_hem()
1315 list_splice(&head->leaf, &hem_list->btm_bt); in setup_root_hem()
1316 list_splice(&head->root, &hem_list->root_bt); in setup_root_hem()
1318 list_splice(&head->branch[i], &hem_list->mid_bt[i][0]); in setup_root_hem()
1324 struct hns_roce_hem_list *hem_list, int unit, in hem_list_alloc_root_bt() argument
1334 root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset); in hem_list_alloc_root_bt()
1339 root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions, in hem_list_alloc_root_bt()
1350 hem_list->root_ba = root_hem->dma_addr; in hem_list_alloc_root_bt()
1351 list_add(&root_hem->list, &head.root); in hem_list_alloc_root_bt()
1352 ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions, in hem_list_alloc_root_bt()
1369 /* construct the base address table and link them by address hop config */
1378 int unit; in hns_roce_hem_list_request() local
1383 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", in hns_roce_hem_list_request()
1385 return -EINVAL; in hns_roce_hem_list_request()
1388 unit = (1 << bt_pg_shift) / BA_BYTE_LEN; in hns_roce_hem_list_request()
1391 if (!r->count) in hns_roce_hem_list_request()
1394 end = r->offset + r->count; in hns_roce_hem_list_request()
1395 for (ofs = r->offset, loop = 1; ofs < end; ofs += unit, loop++) { in hns_roce_hem_list_request()
1399 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, in hns_roce_hem_list_request()
1400 hem_list->mid_bt[i], in hns_roce_hem_list_request()
1401 &hem_list->btm_bt); in hns_roce_hem_list_request()
1403 dev_err(hr_dev->dev, in hns_roce_hem_list_request()
1410 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, in hns_roce_hem_list_request()
1413 dev_err(hr_dev->dev, "alloc hem root fail ret = %d!\n", ret); in hns_roce_hem_list_request()
1430 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j]); in hns_roce_hem_list_release()
1432 hem_list_free_all(hr_dev, &hem_list->root_bt); in hns_roce_hem_list_release()
1433 INIT_LIST_HEAD(&hem_list->btm_bt); in hns_roce_hem_list_release()
1434 hem_list->root_ba = 0; in hns_roce_hem_list_release()
1441 INIT_LIST_HEAD(&hem_list->root_bt); in hns_roce_hem_list_init()
1442 INIT_LIST_HEAD(&hem_list->btm_bt); in hns_roce_hem_list_init()
1445 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); in hns_roce_hem_list_init()
1452 struct list_head *head = &hem_list->btm_bt; in hns_roce_hem_list_find_mtt()
1464 nr = offset - hem->start; in hns_roce_hem_list_find_mtt()
1465 cpu_base = hem->addr + nr * BA_BYTE_LEN; in hns_roce_hem_list_find_mtt()
1466 nr = hem->end + 1 - offset; in hns_roce_hem_list_find_mtt()