Lines Matching +full:pd +full:- +full:node
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
15 * - Redistributions of source code must retain the above
19 * - Redistributions in binary form must reproduce the above
40 #include <linux/dma-buf.h>
41 #include <linux/dma-resv.h>
57 static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
64 struct ib_pd *pd) in set_mkc_access_pd_addr_fields() argument
66 struct mlx5_ib_dev *dev = to_mdev(pd->device); in set_mkc_access_pd_addr_fields()
75 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) in set_mkc_access_pd_addr_fields()
78 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) || in set_mkc_access_pd_addr_fields()
79 (MLX5_CAP_GEN(dev->mdev, in set_mkc_access_pd_addr_fields()
81 pcie_relaxed_ordering_enabled(dev->mdev->pdev))) in set_mkc_access_pd_addr_fields()
85 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); in set_mkc_access_pd_addr_fields()
92 u8 key = atomic_inc_return(&dev->mkey_var); in assign_mkey_variant()
105 assign_mkey_variant(dev, &mkey->key, in); in mlx5_ib_create_mkey()
106 ret = mlx5_core_create_mkey(dev->mdev, &mkey->key, in, inlen); in mlx5_ib_create_mkey()
108 init_waitqueue_head(&mkey->wait); in mlx5_ib_create_mkey()
115 struct mlx5_ib_dev *dev = async_create->ent->dev; in mlx5_ib_create_mkey_cb()
119 MLX5_SET(create_mkey_in, async_create->in, opcode, in mlx5_ib_create_mkey_cb()
121 assign_mkey_variant(dev, &async_create->mkey, async_create->in); in mlx5_ib_create_mkey_cb()
122 return mlx5_cmd_exec_cb(&dev->async_ctx, async_create->in, inlen, in mlx5_ib_create_mkey_cb()
123 async_create->out, outlen, create_mkey_callback, in mlx5_ib_create_mkey_cb()
124 &async_create->cb_work); in mlx5_ib_create_mkey_cb()
132 WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key))); in destroy_mkey()
134 return mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key); in destroy_mkey()
139 if (status == -ENXIO) /* core driver is not available */ in create_mkey_warn()
143 if (status != -EREMOTEIO) /* driver specific failure */ in create_mkey_warn()
147 mlx5_cmd_out_err(dev->mdev, MLX5_CMD_OP_CREATE_MKEY, 0, out); in create_mkey_warn()
152 unsigned long tmp = ent->mkeys_queue.ci % NUM_MKEYS_PER_PAGE; in push_mkey_locked()
155 lockdep_assert_held(&ent->mkeys_queue.lock); in push_mkey_locked()
156 if (ent->mkeys_queue.ci >= in push_mkey_locked()
157 ent->mkeys_queue.num_pages * NUM_MKEYS_PER_PAGE) { in push_mkey_locked()
160 return -ENOMEM; in push_mkey_locked()
161 ent->mkeys_queue.num_pages++; in push_mkey_locked()
162 list_add_tail(&page->list, &ent->mkeys_queue.pages_list); in push_mkey_locked()
164 page = list_last_entry(&ent->mkeys_queue.pages_list, in push_mkey_locked()
168 page->mkeys[tmp] = mkey; in push_mkey_locked()
169 ent->mkeys_queue.ci++; in push_mkey_locked()
175 unsigned long tmp = (ent->mkeys_queue.ci - 1) % NUM_MKEYS_PER_PAGE; in pop_mkey_locked()
179 lockdep_assert_held(&ent->mkeys_queue.lock); in pop_mkey_locked()
180 last_page = list_last_entry(&ent->mkeys_queue.pages_list, in pop_mkey_locked()
182 mkey = last_page->mkeys[tmp]; in pop_mkey_locked()
183 last_page->mkeys[tmp] = 0; in pop_mkey_locked()
184 ent->mkeys_queue.ci--; in pop_mkey_locked()
185 if (ent->mkeys_queue.num_pages > 1 && !tmp) { in pop_mkey_locked()
186 list_del(&last_page->list); in pop_mkey_locked()
187 ent->mkeys_queue.num_pages--; in pop_mkey_locked()
197 struct mlx5_cache_ent *ent = mkey_out->ent; in create_mkey_callback()
198 struct mlx5_ib_dev *dev = ent->dev; in create_mkey_callback()
202 create_mkey_warn(dev, status, mkey_out->out); in create_mkey_callback()
204 spin_lock_irqsave(&ent->mkeys_queue.lock, flags); in create_mkey_callback()
205 ent->pending--; in create_mkey_callback()
206 WRITE_ONCE(dev->fill_delay, 1); in create_mkey_callback()
207 spin_unlock_irqrestore(&ent->mkeys_queue.lock, flags); in create_mkey_callback()
208 mod_timer(&dev->delay_timer, jiffies + HZ); in create_mkey_callback()
212 mkey_out->mkey |= mlx5_idx_to_mkey( in create_mkey_callback()
213 MLX5_GET(create_mkey_out, mkey_out->out, mkey_index)); in create_mkey_callback()
214 WRITE_ONCE(dev->cache.last_add, jiffies); in create_mkey_callback()
216 spin_lock_irqsave(&ent->mkeys_queue.lock, flags); in create_mkey_callback()
217 push_mkey_locked(ent, mkey_out->mkey); in create_mkey_callback()
218 ent->pending--; in create_mkey_callback()
221 spin_unlock_irqrestore(&ent->mkeys_queue.lock, flags); in create_mkey_callback()
246 set_mkc_access_pd_addr_fields(mkc, ent->rb_key.access_flags, 0, in set_cache_mkc()
247 ent->dev->umrc.pd); in set_cache_mkc()
250 MLX5_SET(mkc, mkc, access_mode_1_0, ent->rb_key.access_mode & 0x3); in set_cache_mkc()
252 (ent->rb_key.access_mode >> 2) & 0x7); in set_cache_mkc()
253 MLX5_SET(mkc, mkc, ma_translation_mode, !!ent->rb_key.ats); in set_cache_mkc()
256 get_mkc_octo_size(ent->rb_key.access_mode, in set_cache_mkc()
257 ent->rb_key.ndescs)); in set_cache_mkc()
273 return -ENOMEM; in add_keys()
274 mkc = MLX5_ADDR_OF(create_mkey_in, async_create->in, in add_keys()
277 async_create->ent = ent; in add_keys()
279 spin_lock_irq(&ent->mkeys_queue.lock); in add_keys()
280 if (ent->pending >= MAX_PENDING_REG_MR) { in add_keys()
281 err = -EAGAIN; in add_keys()
284 ent->pending++; in add_keys()
285 spin_unlock_irq(&ent->mkeys_queue.lock); in add_keys()
289 mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err); in add_keys()
297 spin_lock_irq(&ent->mkeys_queue.lock); in add_keys()
298 ent->pending--; in add_keys()
300 spin_unlock_irq(&ent->mkeys_queue.lock); in add_keys()
315 return -ENOMEM; in create_cache_mkey()
319 err = mlx5_core_create_mkey(ent->dev->mdev, mkey, in, inlen); in create_cache_mkey()
323 WRITE_ONCE(ent->dev->cache.last_add, jiffies); in create_cache_mkey()
333 lockdep_assert_held(&ent->mkeys_queue.lock); in remove_cache_mr_locked()
334 if (!ent->mkeys_queue.ci) in remove_cache_mr_locked()
337 spin_unlock_irq(&ent->mkeys_queue.lock); in remove_cache_mr_locked()
338 mlx5_core_destroy_mkey(ent->dev->mdev, mkey); in remove_cache_mr_locked()
339 spin_lock_irq(&ent->mkeys_queue.lock); in remove_cache_mr_locked()
344 __acquires(&ent->mkeys_queue.lock) __releases(&ent->mkeys_queue.lock) in resize_available_mrs()
348 lockdep_assert_held(&ent->mkeys_queue.lock); in resize_available_mrs()
352 target = ent->limit * 2; in resize_available_mrs()
353 if (target == ent->pending + ent->mkeys_queue.ci) in resize_available_mrs()
355 if (target > ent->pending + ent->mkeys_queue.ci) { in resize_available_mrs()
356 u32 todo = target - (ent->pending + ent->mkeys_queue.ci); in resize_available_mrs()
358 spin_unlock_irq(&ent->mkeys_queue.lock); in resize_available_mrs()
360 if (err == -EAGAIN) in resize_available_mrs()
362 spin_lock_irq(&ent->mkeys_queue.lock); in resize_available_mrs()
364 if (err != -EAGAIN) in resize_available_mrs()
377 struct mlx5_cache_ent *ent = filp->private_data; in size_write()
390 spin_lock_irq(&ent->mkeys_queue.lock); in size_write()
391 if (target < ent->in_use) { in size_write()
392 err = -EINVAL; in size_write()
395 target = target - ent->in_use; in size_write()
396 if (target < ent->limit || target > ent->limit*2) { in size_write()
397 err = -EINVAL; in size_write()
403 spin_unlock_irq(&ent->mkeys_queue.lock); in size_write()
408 spin_unlock_irq(&ent->mkeys_queue.lock); in size_write()
415 struct mlx5_cache_ent *ent = filp->private_data; in size_read()
420 ent->mkeys_queue.ci + ent->in_use); in size_read()
437 struct mlx5_cache_ent *ent = filp->private_data; in limit_write()
449 spin_lock_irq(&ent->mkeys_queue.lock); in limit_write()
450 ent->limit = var; in limit_write()
452 spin_unlock_irq(&ent->mkeys_queue.lock); in limit_write()
461 struct mlx5_cache_ent *ent = filp->private_data; in limit_read()
465 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit); in limit_read()
482 struct rb_node *node; in someone_adding() local
485 mutex_lock(&cache->rb_lock); in someone_adding()
486 for (node = rb_first(&cache->rb_root); node; node = rb_next(node)) { in someone_adding()
487 ent = rb_entry(node, struct mlx5_cache_ent, node); in someone_adding()
488 spin_lock_irq(&ent->mkeys_queue.lock); in someone_adding()
489 ret = ent->mkeys_queue.ci < ent->limit; in someone_adding()
490 spin_unlock_irq(&ent->mkeys_queue.lock); in someone_adding()
492 mutex_unlock(&cache->rb_lock); in someone_adding()
496 mutex_unlock(&cache->rb_lock); in someone_adding()
507 lockdep_assert_held(&ent->mkeys_queue.lock); in queue_adjust_cache_locked()
509 if (ent->disabled || READ_ONCE(ent->dev->fill_delay) || ent->is_tmp) in queue_adjust_cache_locked()
511 if (ent->mkeys_queue.ci < ent->limit) { in queue_adjust_cache_locked()
512 ent->fill_to_high_water = true; in queue_adjust_cache_locked()
513 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0); in queue_adjust_cache_locked()
514 } else if (ent->fill_to_high_water && in queue_adjust_cache_locked()
515 ent->mkeys_queue.ci + ent->pending < 2 * ent->limit) { in queue_adjust_cache_locked()
520 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0); in queue_adjust_cache_locked()
521 } else if (ent->mkeys_queue.ci == 2 * ent->limit) { in queue_adjust_cache_locked()
522 ent->fill_to_high_water = false; in queue_adjust_cache_locked()
523 } else if (ent->mkeys_queue.ci > 2 * ent->limit) { in queue_adjust_cache_locked()
525 ent->fill_to_high_water = false; in queue_adjust_cache_locked()
526 if (ent->pending) in queue_adjust_cache_locked()
527 queue_delayed_work(ent->dev->cache.wq, &ent->dwork, in queue_adjust_cache_locked()
530 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0); in queue_adjust_cache_locked()
538 spin_lock_irq(&ent->mkeys_queue.lock); in clean_keys()
539 while (ent->mkeys_queue.ci) { in clean_keys()
541 spin_unlock_irq(&ent->mkeys_queue.lock); in clean_keys()
542 mlx5_core_destroy_mkey(dev->mdev, mkey); in clean_keys()
543 spin_lock_irq(&ent->mkeys_queue.lock); in clean_keys()
545 ent->tmp_cleanup_scheduled = false; in clean_keys()
546 spin_unlock_irq(&ent->mkeys_queue.lock); in clean_keys()
551 struct mlx5_ib_dev *dev = ent->dev; in __cache_work_func()
552 struct mlx5_mkey_cache *cache = &dev->cache; in __cache_work_func()
555 spin_lock_irq(&ent->mkeys_queue.lock); in __cache_work_func()
556 if (ent->disabled) in __cache_work_func()
559 if (ent->fill_to_high_water && in __cache_work_func()
560 ent->mkeys_queue.ci + ent->pending < 2 * ent->limit && in __cache_work_func()
561 !READ_ONCE(dev->fill_delay)) { in __cache_work_func()
562 spin_unlock_irq(&ent->mkeys_queue.lock); in __cache_work_func()
564 spin_lock_irq(&ent->mkeys_queue.lock); in __cache_work_func()
565 if (ent->disabled) in __cache_work_func()
573 if (err != -EAGAIN) { in __cache_work_func()
578 queue_delayed_work(cache->wq, &ent->dwork, in __cache_work_func()
582 } else if (ent->mkeys_queue.ci > 2 * ent->limit) { in __cache_work_func()
597 spin_unlock_irq(&ent->mkeys_queue.lock); in __cache_work_func()
600 READ_ONCE(cache->last_add) + 300 * HZ); in __cache_work_func()
601 spin_lock_irq(&ent->mkeys_queue.lock); in __cache_work_func()
602 if (ent->disabled) in __cache_work_func()
605 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ); in __cache_work_func()
612 spin_unlock_irq(&ent->mkeys_queue.lock); in __cache_work_func()
621 if (ent->is_tmp) in delayed_cache_work_func()
622 clean_keys(ent->dev, ent); in delayed_cache_work_func()
632 res = key1.ats - key2.ats; in cache_ent_key_cmp()
636 res = key1.access_mode - key2.access_mode; in cache_ent_key_cmp()
640 res = key1.access_flags - key2.access_flags; in cache_ent_key_cmp()
649 return key1.ndescs - key2.ndescs; in cache_ent_key_cmp()
655 struct rb_node **new = &cache->rb_root.rb_node, *parent = NULL; in mlx5_cache_ent_insert()
659 /* Figure out where to put new node */ in mlx5_cache_ent_insert()
661 cur = rb_entry(*new, struct mlx5_cache_ent, node); in mlx5_cache_ent_insert()
663 cmp = cache_ent_key_cmp(cur->rb_key, ent->rb_key); in mlx5_cache_ent_insert()
665 new = &((*new)->rb_left); in mlx5_cache_ent_insert()
667 new = &((*new)->rb_right); in mlx5_cache_ent_insert()
669 return -EEXIST; in mlx5_cache_ent_insert()
672 /* Add new node and rebalance tree. */ in mlx5_cache_ent_insert()
673 rb_link_node(&ent->node, parent, new); in mlx5_cache_ent_insert()
674 rb_insert_color(&ent->node, &cache->rb_root); in mlx5_cache_ent_insert()
683 struct rb_node *node = dev->cache.rb_root.rb_node; in mkey_cache_ent_from_rb_key() local
691 while (node) { in mkey_cache_ent_from_rb_key()
692 cur = rb_entry(node, struct mlx5_cache_ent, node); in mkey_cache_ent_from_rb_key()
693 cmp = cache_ent_key_cmp(cur->rb_key, rb_key); in mkey_cache_ent_from_rb_key()
696 node = node->rb_left; in mkey_cache_ent_from_rb_key()
699 node = node->rb_right; in mkey_cache_ent_from_rb_key()
712 smallest->rb_key.access_mode == rb_key.access_mode && in mkey_cache_ent_from_rb_key()
713 smallest->rb_key.access_flags == rb_key.access_flags && in mkey_cache_ent_from_rb_key()
714 smallest->rb_key.ats == rb_key.ats && in mkey_cache_ent_from_rb_key()
715 smallest->rb_key.ndescs <= ndescs_limit) ? in mkey_cache_ent_from_rb_key()
729 return ERR_PTR(-ENOMEM); in _mlx5_mr_cache_alloc()
731 spin_lock_irq(&ent->mkeys_queue.lock); in _mlx5_mr_cache_alloc()
732 ent->in_use++; in _mlx5_mr_cache_alloc()
734 if (!ent->mkeys_queue.ci) { in _mlx5_mr_cache_alloc()
736 ent->miss++; in _mlx5_mr_cache_alloc()
737 spin_unlock_irq(&ent->mkeys_queue.lock); in _mlx5_mr_cache_alloc()
738 err = create_cache_mkey(ent, &mr->mmkey.key); in _mlx5_mr_cache_alloc()
740 spin_lock_irq(&ent->mkeys_queue.lock); in _mlx5_mr_cache_alloc()
741 ent->in_use--; in _mlx5_mr_cache_alloc()
742 spin_unlock_irq(&ent->mkeys_queue.lock); in _mlx5_mr_cache_alloc()
747 mr->mmkey.key = pop_mkey_locked(ent); in _mlx5_mr_cache_alloc()
749 spin_unlock_irq(&ent->mkeys_queue.lock); in _mlx5_mr_cache_alloc()
751 mr->mmkey.cache_ent = ent; in _mlx5_mr_cache_alloc()
752 mr->mmkey.type = MLX5_MKEY_MR; in _mlx5_mr_cache_alloc()
753 mr->mmkey.rb_key = ent->rb_key; in _mlx5_mr_cache_alloc()
754 mr->mmkey.cacheable = true; in _mlx5_mr_cache_alloc()
755 init_waitqueue_head(&mr->mmkey.wait); in _mlx5_mr_cache_alloc()
765 MLX5_CAP_GEN(dev->mdev, atomic) && in get_unchangeable_access_flags()
766 MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)) in get_unchangeable_access_flags()
770 MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) && in get_unchangeable_access_flags()
771 !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)) in get_unchangeable_access_flags()
775 (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) || in get_unchangeable_access_flags()
776 MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_pci_enabled)) && in get_unchangeable_access_flags()
777 !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) in get_unchangeable_access_flags()
795 return ERR_PTR(-EOPNOTSUPP); in mlx5_mr_cache_alloc()
802 if (!mlx5_debugfs_root || dev->is_rep) in mlx5_mkey_cache_debugfs_cleanup()
805 debugfs_remove_recursive(dev->cache.fs_root); in mlx5_mkey_cache_debugfs_cleanup()
806 dev->cache.fs_root = NULL; in mlx5_mkey_cache_debugfs_cleanup()
812 int order = order_base_2(ent->rb_key.ndescs); in mlx5_mkey_cache_debugfs_add_ent()
815 if (!mlx5_debugfs_root || dev->is_rep) in mlx5_mkey_cache_debugfs_add_ent()
818 if (ent->rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM) in mlx5_mkey_cache_debugfs_add_ent()
821 sprintf(ent->name, "%d", order); in mlx5_mkey_cache_debugfs_add_ent()
822 dir = debugfs_create_dir(ent->name, dev->cache.fs_root); in mlx5_mkey_cache_debugfs_add_ent()
825 debugfs_create_ulong("cur", 0400, dir, &ent->mkeys_queue.ci); in mlx5_mkey_cache_debugfs_add_ent()
826 debugfs_create_u32("miss", 0600, dir, &ent->miss); in mlx5_mkey_cache_debugfs_add_ent()
831 struct dentry *dbg_root = mlx5_debugfs_get_dev_root(dev->mdev); in mlx5_mkey_cache_debugfs_init()
832 struct mlx5_mkey_cache *cache = &dev->cache; in mlx5_mkey_cache_debugfs_init()
834 if (!mlx5_debugfs_root || dev->is_rep) in mlx5_mkey_cache_debugfs_init()
837 cache->fs_root = debugfs_create_dir("mr_cache", dbg_root); in mlx5_mkey_cache_debugfs_init()
844 WRITE_ONCE(dev->fill_delay, 0); in delay_time_func()
853 return -ENOMEM; in mlx5r_mkeys_init()
854 INIT_LIST_HEAD(&ent->mkeys_queue.pages_list); in mlx5r_mkeys_init()
855 spin_lock_init(&ent->mkeys_queue.lock); in mlx5r_mkeys_init()
856 list_add_tail(&page->list, &ent->mkeys_queue.pages_list); in mlx5r_mkeys_init()
857 ent->mkeys_queue.num_pages++; in mlx5r_mkeys_init()
865 WARN_ON(ent->mkeys_queue.ci || ent->mkeys_queue.num_pages > 1); in mlx5r_mkeys_uninit()
866 page = list_last_entry(&ent->mkeys_queue.pages_list, in mlx5r_mkeys_uninit()
868 list_del(&page->list); in mlx5r_mkeys_uninit()
883 return ERR_PTR(-ENOMEM); in mlx5r_cache_create_ent_locked()
888 ent->rb_key = rb_key; in mlx5r_cache_create_ent_locked()
889 ent->dev = dev; in mlx5r_cache_create_ent_locked()
890 ent->is_tmp = !persistent_entry; in mlx5r_cache_create_ent_locked()
892 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); in mlx5r_cache_create_ent_locked()
894 ret = mlx5_cache_ent_insert(&dev->cache, ent); in mlx5r_cache_create_ent_locked()
902 order = order_base_2(rb_key.ndescs) - 2; in mlx5r_cache_create_ent_locked()
904 if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) && in mlx5r_cache_create_ent_locked()
905 !dev->is_rep && mlx5_core_is_pf(dev->mdev) && in mlx5r_cache_create_ent_locked()
907 ent->limit = dev->mdev->profile.mr_cache[order].limit; in mlx5r_cache_create_ent_locked()
909 ent->limit = 0; in mlx5r_cache_create_ent_locked()
924 struct rb_root *root = &dev->cache.rb_root; in mlx5r_destroy_cache_entries()
926 struct rb_node *node; in mlx5r_destroy_cache_entries() local
928 mutex_lock(&dev->cache.rb_lock); in mlx5r_destroy_cache_entries()
929 node = rb_first(root); in mlx5r_destroy_cache_entries()
930 while (node) { in mlx5r_destroy_cache_entries()
931 ent = rb_entry(node, struct mlx5_cache_ent, node); in mlx5r_destroy_cache_entries()
932 node = rb_next(node); in mlx5r_destroy_cache_entries()
934 rb_erase(&ent->node, root); in mlx5r_destroy_cache_entries()
938 mutex_unlock(&dev->cache.rb_lock); in mlx5r_destroy_cache_entries()
943 struct mlx5_mkey_cache *cache = &dev->cache; in mlx5_mkey_cache_init()
944 struct rb_root *root = &dev->cache.rb_root; in mlx5_mkey_cache_init()
949 struct rb_node *node; in mlx5_mkey_cache_init() local
953 mutex_init(&dev->slow_path_mutex); in mlx5_mkey_cache_init()
954 mutex_init(&dev->cache.rb_lock); in mlx5_mkey_cache_init()
955 dev->cache.rb_root = RB_ROOT; in mlx5_mkey_cache_init()
956 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM); in mlx5_mkey_cache_init()
957 if (!cache->wq) { in mlx5_mkey_cache_init()
959 return -ENOMEM; in mlx5_mkey_cache_init()
962 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx); in mlx5_mkey_cache_init()
963 timer_setup(&dev->delay_timer, delay_time_func, 0); in mlx5_mkey_cache_init()
965 mutex_lock(&cache->rb_lock); in mlx5_mkey_cache_init()
979 mutex_unlock(&cache->rb_lock); in mlx5_mkey_cache_init()
980 for (node = rb_first(root); node; node = rb_next(node)) { in mlx5_mkey_cache_init()
981 ent = rb_entry(node, struct mlx5_cache_ent, node); in mlx5_mkey_cache_init()
982 spin_lock_irq(&ent->mkeys_queue.lock); in mlx5_mkey_cache_init()
984 spin_unlock_irq(&ent->mkeys_queue.lock); in mlx5_mkey_cache_init()
990 mutex_unlock(&cache->rb_lock); in mlx5_mkey_cache_init()
993 destroy_workqueue(cache->wq); in mlx5_mkey_cache_init()
1000 struct rb_root *root = &dev->cache.rb_root; in mlx5_mkey_cache_cleanup()
1002 struct rb_node *node; in mlx5_mkey_cache_cleanup() local
1004 if (!dev->cache.wq) in mlx5_mkey_cache_cleanup()
1007 mutex_lock(&dev->cache.rb_lock); in mlx5_mkey_cache_cleanup()
1008 for (node = rb_first(root); node; node = rb_next(node)) { in mlx5_mkey_cache_cleanup()
1009 ent = rb_entry(node, struct mlx5_cache_ent, node); in mlx5_mkey_cache_cleanup()
1010 spin_lock_irq(&ent->mkeys_queue.lock); in mlx5_mkey_cache_cleanup()
1011 ent->disabled = true; in mlx5_mkey_cache_cleanup()
1012 spin_unlock_irq(&ent->mkeys_queue.lock); in mlx5_mkey_cache_cleanup()
1013 cancel_delayed_work(&ent->dwork); in mlx5_mkey_cache_cleanup()
1015 mutex_unlock(&dev->cache.rb_lock); in mlx5_mkey_cache_cleanup()
1021 flush_workqueue(dev->cache.wq); in mlx5_mkey_cache_cleanup()
1024 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx); in mlx5_mkey_cache_cleanup()
1029 destroy_workqueue(dev->cache.wq); in mlx5_mkey_cache_cleanup()
1030 del_timer_sync(&dev->delay_timer); in mlx5_mkey_cache_cleanup()
1033 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) in mlx5_ib_get_dma_mr() argument
1035 struct mlx5_ib_dev *dev = to_mdev(pd->device); in mlx5_ib_get_dma_mr()
1044 return ERR_PTR(-ENOMEM); in mlx5_ib_get_dma_mr()
1048 err = -ENOMEM; in mlx5_ib_get_dma_mr()
1057 pd); in mlx5_ib_get_dma_mr()
1058 MLX5_SET(mkc, mkc, ma_translation_mode, MLX5_CAP_GEN(dev->mdev, ats)); in mlx5_ib_get_dma_mr()
1060 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); in mlx5_ib_get_dma_mr()
1065 mr->mmkey.type = MLX5_MKEY_MR; in mlx5_ib_get_dma_mr()
1066 mr->ibmr.lkey = mr->mmkey.key; in mlx5_ib_get_dma_mr()
1067 mr->ibmr.rkey = mr->mmkey.key; in mlx5_ib_get_dma_mr()
1068 mr->umem = NULL; in mlx5_ib_get_dma_mr()
1070 return &mr->ibmr; in mlx5_ib_get_dma_mr()
1087 offset = addr & (page_size - 1); in get_octo_len()
1094 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) in mkey_cache_max_order()
1102 mr->ibmr.lkey = mr->mmkey.key; in set_mr_fields()
1103 mr->ibmr.rkey = mr->mmkey.key; in set_mr_fields()
1104 mr->ibmr.length = length; in set_mr_fields()
1105 mr->ibmr.device = &dev->ib_dev; in set_mr_fields()
1106 mr->ibmr.iova = iova; in set_mr_fields()
1107 mr->access_flags = access_flags; in set_mr_fields()
1117 umem->iova = iova; in mlx5_umem_dmabuf_default_pgsz()
1121 static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, in alloc_cacheable_mr() argument
1125 struct mlx5_ib_dev *dev = to_mdev(pd->device); in alloc_cacheable_mr()
1131 if (umem->is_dmabuf) in alloc_cacheable_mr()
1136 return ERR_PTR(-EINVAL); in alloc_cacheable_mr()
1148 mutex_lock(&dev->slow_path_mutex); in alloc_cacheable_mr()
1149 mr = reg_create(pd, umem, iova, access_flags, page_size, false, access_mode); in alloc_cacheable_mr()
1150 mutex_unlock(&dev->slow_path_mutex); in alloc_cacheable_mr()
1153 mr->mmkey.rb_key = rb_key; in alloc_cacheable_mr()
1154 mr->mmkey.cacheable = true; in alloc_cacheable_mr()
1162 mr->ibmr.pd = pd; in alloc_cacheable_mr()
1163 mr->umem = umem; in alloc_cacheable_mr()
1164 mr->page_shift = order_base_2(page_size); in alloc_cacheable_mr()
1165 set_mr_fields(dev, mr, umem->length, access_flags, iova); in alloc_cacheable_mr()
1171 reg_create_crossing_vhca_mr(struct ib_pd *pd, u64 iova, u64 length, int access_flags, in reg_create_crossing_vhca_mr() argument
1174 struct mlx5_ib_dev *dev = to_mdev(pd->device); in reg_create_crossing_vhca_mr()
1182 if (!MLX5_CAP_GEN(dev->mdev, crossing_vhca_mkey)) in reg_create_crossing_vhca_mr()
1183 return ERR_PTR(-EOPNOTSUPP); in reg_create_crossing_vhca_mr()
1187 return ERR_PTR(-ENOMEM); in reg_create_crossing_vhca_mr()
1192 err = -ENOMEM; in reg_create_crossing_vhca_mr()
1198 MLX5_CAP_GEN(dev->mdev, vhca_id)); in reg_create_crossing_vhca_mr()
1204 set_mkc_access_pd_addr_fields(mkc, access_flags, 0, pd); in reg_create_crossing_vhca_mr()
1209 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); in reg_create_crossing_vhca_mr()
1213 mr->mmkey.type = MLX5_MKEY_MR; in reg_create_crossing_vhca_mr()
1215 mr->ibmr.pd = pd; in reg_create_crossing_vhca_mr()
1217 mlx5_ib_dbg(dev, "crossing mkey = 0x%x\n", mr->mmkey.key); in reg_create_crossing_vhca_mr()
1219 return &mr->ibmr; in reg_create_crossing_vhca_mr()
1231 static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, in reg_create() argument
1236 struct mlx5_ib_dev *dev = to_mdev(pd->device); in reg_create()
1243 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)) && in reg_create()
1248 return ERR_PTR(-EINVAL); in reg_create()
1251 return ERR_PTR(-ENOMEM); in reg_create()
1253 mr->ibmr.pd = pd; in reg_create()
1254 mr->access_flags = access_flags; in reg_create()
1255 mr->page_shift = order_base_2(page_size); in reg_create()
1263 err = -ENOMEM; in reg_create()
1269 err = -EINVAL; in reg_create()
1272 mlx5_ib_populate_pas(umem, 1UL << mr->page_shift, pas, in reg_create()
1283 populate ? pd : dev->umrc.pd); in reg_create()
1284 /* In case a data direct flow, overwrite the pdn field by its internal kernel PD */ in reg_create()
1285 if (umem->is_dmabuf && ksm_mode) in reg_create()
1286 MLX5_SET(mkc, mkc, pd, dev->ddr.pdn); in reg_create()
1292 MLX5_SET64(mkc, mkc, len, umem->length); in reg_create()
1296 get_octo_len(iova, umem->length, mr->page_shift) * 2); in reg_create()
1299 get_octo_len(iova, umem->length, mr->page_shift)); in reg_create()
1300 MLX5_SET(mkc, mkc, log_page_size, mr->page_shift); in reg_create()
1305 get_octo_len(iova, umem->length, mr->page_shift)); in reg_create()
1308 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); in reg_create()
1313 mr->mmkey.type = MLX5_MKEY_MR; in reg_create()
1314 mr->mmkey.ndescs = get_octo_len(iova, umem->length, mr->page_shift); in reg_create()
1315 mr->umem = umem; in reg_create()
1316 set_mr_fields(dev, mr, umem->length, access_flags, iova); in reg_create()
1319 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key); in reg_create()
1330 static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr, in mlx5_ib_get_dm_mr() argument
1333 struct mlx5_ib_dev *dev = to_mdev(pd->device); in mlx5_ib_get_dm_mr()
1342 return ERR_PTR(-ENOMEM); in mlx5_ib_get_dm_mr()
1346 err = -ENOMEM; in mlx5_ib_get_dm_mr()
1355 set_mkc_access_pd_addr_fields(mkc, acc, start_addr, pd); in mlx5_ib_get_dm_mr()
1357 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); in mlx5_ib_get_dm_mr()
1365 return &mr->ibmr; in mlx5_ib_get_dm_mr()
1376 int mlx5_ib_advise_mr(struct ib_pd *pd, in mlx5_ib_advise_mr() argument
1386 return -EOPNOTSUPP; in mlx5_ib_advise_mr()
1388 return mlx5_ib_advise_mr_prefetch(pd, advice, flags, in mlx5_ib_advise_mr()
1392 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, in mlx5_ib_reg_dm_mr() argument
1397 struct mlx5_core_dev *dev = to_mdev(dm->device)->mdev; in mlx5_ib_reg_dm_mr()
1398 u64 start_addr = mdm->dev_addr + attr->offset; in mlx5_ib_reg_dm_mr()
1401 switch (mdm->type) { in mlx5_ib_reg_dm_mr()
1403 if (attr->access_flags & ~MLX5_IB_DM_MEMIC_ALLOWED_ACCESS) in mlx5_ib_reg_dm_mr()
1404 return ERR_PTR(-EINVAL); in mlx5_ib_reg_dm_mr()
1407 start_addr -= pci_resource_start(dev->pdev, 0); in mlx5_ib_reg_dm_mr()
1413 if (attr->access_flags & ~MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS) in mlx5_ib_reg_dm_mr()
1414 return ERR_PTR(-EINVAL); in mlx5_ib_reg_dm_mr()
1419 return ERR_PTR(-EINVAL); in mlx5_ib_reg_dm_mr()
1422 return mlx5_ib_get_dm_mr(pd, start_addr, attr->length, in mlx5_ib_reg_dm_mr()
1423 attr->access_flags, mode); in mlx5_ib_reg_dm_mr()
1426 static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem, in create_real_mr() argument
1429 struct mlx5_ib_dev *dev = to_mdev(pd->device); in create_real_mr()
1434 xlt_with_umr = mlx5r_umr_can_load_pas(dev, umem->length); in create_real_mr()
1436 mr = alloc_cacheable_mr(pd, umem, iova, access_flags, in create_real_mr()
1442 mutex_lock(&dev->slow_path_mutex); in create_real_mr()
1443 mr = reg_create(pd, umem, iova, access_flags, page_size, in create_real_mr()
1445 mutex_unlock(&dev->slow_path_mutex); in create_real_mr()
1452 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); in create_real_mr()
1454 atomic_add(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages); in create_real_mr()
1464 mlx5_ib_dereg_mr(&mr->ibmr, NULL); in create_real_mr()
1468 return &mr->ibmr; in create_real_mr()
1471 static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length, in create_user_odp_mr() argument
1475 struct mlx5_ib_dev *dev = to_mdev(pd->device); in create_user_odp_mr()
1481 return ERR_PTR(-EOPNOTSUPP); in create_user_odp_mr()
1483 err = mlx5r_odp_create_eq(dev, &dev->odp_pf_eq); in create_user_odp_mr()
1488 return ERR_PTR(-EINVAL); in create_user_odp_mr()
1489 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) in create_user_odp_mr()
1490 return ERR_PTR(-EINVAL); in create_user_odp_mr()
1492 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), access_flags); in create_user_odp_mr()
1495 return &mr->ibmr; in create_user_odp_mr()
1500 return ERR_PTR(-EINVAL); in create_user_odp_mr()
1502 odp = ib_umem_odp_get(&dev->ib_dev, start, length, access_flags, in create_user_odp_mr()
1507 mr = alloc_cacheable_mr(pd, &odp->umem, iova, access_flags, in create_user_odp_mr()
1510 ib_umem_release(&odp->umem); in create_user_odp_mr()
1513 xa_init(&mr->implicit_children); in create_user_odp_mr()
1515 odp->private = mr; in create_user_odp_mr()
1516 err = mlx5r_store_odp_mkey(dev, &mr->mmkey); in create_user_odp_mr()
1523 return &mr->ibmr; in create_user_odp_mr()
1526 mlx5_ib_dereg_mr(&mr->ibmr, NULL); in create_user_odp_mr()
1530 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, in mlx5_ib_reg_user_mr() argument
1534 struct mlx5_ib_dev *dev = to_mdev(pd->device); in mlx5_ib_reg_user_mr()
1539 return ERR_PTR(-EOPNOTSUPP); in mlx5_ib_reg_user_mr()
1549 return create_user_odp_mr(pd, start, length, iova, access_flags, in mlx5_ib_reg_user_mr()
1551 umem = ib_umem_get(&dev->ib_dev, start, length, access_flags); in mlx5_ib_reg_user_mr()
1554 return create_real_mr(pd, umem, iova, access_flags); in mlx5_ib_reg_user_mr()
1559 struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv; in mlx5_ib_dmabuf_invalidate_cb()
1560 struct mlx5_ib_mr *mr = umem_dmabuf->private; in mlx5_ib_dmabuf_invalidate_cb()
1562 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); in mlx5_ib_dmabuf_invalidate_cb()
1564 if (!umem_dmabuf->sgt || !mr) in mlx5_ib_dmabuf_invalidate_cb()
1577 reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device, in reg_user_mr_dmabuf() argument
1582 struct mlx5_ib_dev *dev = to_mdev(pd->device); in reg_user_mr_dmabuf()
1592 umem_dmabuf = ib_umem_dmabuf_get(&dev->ib_dev, in reg_user_mr_dmabuf()
1597 umem_dmabuf = ib_umem_dmabuf_get_pinned_with_dma_device(&dev->ib_dev, in reg_user_mr_dmabuf()
1607 mr = alloc_cacheable_mr(pd, &umem_dmabuf->umem, virt_addr, in reg_user_mr_dmabuf()
1610 ib_umem_release(&umem_dmabuf->umem); in reg_user_mr_dmabuf()
1614 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); in reg_user_mr_dmabuf()
1616 atomic_add(ib_umem_num_pages(mr->umem), &dev->mdev->priv.reg_pages); in reg_user_mr_dmabuf()
1617 umem_dmabuf->private = mr; in reg_user_mr_dmabuf()
1619 err = mlx5r_store_odp_mkey(dev, &mr->mmkey); in reg_user_mr_dmabuf()
1623 mr->data_direct = true; in reg_user_mr_dmabuf()
1629 return &mr->ibmr; in reg_user_mr_dmabuf()
1632 __mlx5_ib_dereg_mr(&mr->ibmr); in reg_user_mr_dmabuf()
1637 reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, in reg_user_mr_dmabuf_by_data_direct() argument
1641 struct mlx5_ib_dev *dev = to_mdev(pd->device); in reg_user_mr_dmabuf_by_data_direct()
1649 return ERR_PTR(-EOPNOTSUPP); in reg_user_mr_dmabuf_by_data_direct()
1651 mutex_lock(&dev->data_direct_lock); in reg_user_mr_dmabuf_by_data_direct()
1652 data_direct_dev = dev->data_direct_dev; in reg_user_mr_dmabuf_by_data_direct()
1654 ret = -EINVAL; in reg_user_mr_dmabuf_by_data_direct()
1663 crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev, in reg_user_mr_dmabuf_by_data_direct()
1671 mutex_lock(&dev->slow_path_mutex); in reg_user_mr_dmabuf_by_data_direct()
1672 crossing_mr = reg_create_crossing_vhca_mr(pd, virt_addr, length, access_flags, in reg_user_mr_dmabuf_by_data_direct()
1673 crossed_mr->lkey); in reg_user_mr_dmabuf_by_data_direct()
1674 mutex_unlock(&dev->slow_path_mutex); in reg_user_mr_dmabuf_by_data_direct()
1681 list_add_tail(&to_mmr(crossed_mr)->dd_node, &dev->data_direct_mr_list); in reg_user_mr_dmabuf_by_data_direct()
1682 to_mmr(crossing_mr)->dd_crossed_mr = to_mmr(crossed_mr); in reg_user_mr_dmabuf_by_data_direct()
1683 to_mmr(crossing_mr)->data_direct = true; in reg_user_mr_dmabuf_by_data_direct()
1685 mutex_unlock(&dev->data_direct_lock); in reg_user_mr_dmabuf_by_data_direct()
1689 struct ib_mr *mlx5_ib_reg_user_mr_dmabuf(struct ib_pd *pd, u64 offset, in mlx5_ib_reg_user_mr_dmabuf() argument
1694 struct mlx5_ib_dev *dev = to_mdev(pd->device); in mlx5_ib_reg_user_mr_dmabuf()
1700 return ERR_PTR(-EOPNOTSUPP); in mlx5_ib_reg_user_mr_dmabuf()
1716 return ERR_PTR(-EINVAL); in mlx5_ib_reg_user_mr_dmabuf()
1719 return reg_user_mr_dmabuf_by_data_direct(pd, offset, length, virt_addr, in mlx5_ib_reg_user_mr_dmabuf()
1722 return reg_user_mr_dmabuf(pd, pd->device->dma_device, in mlx5_ib_reg_user_mr_dmabuf()
1750 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); in can_use_umr_rereg_pas()
1753 if (!mr->mmkey.cache_ent) in can_use_umr_rereg_pas()
1755 if (!mlx5r_umr_can_load_pas(dev, new_umem->length)) in can_use_umr_rereg_pas()
1761 return (mr->mmkey.cache_ent->rb_key.ndescs) >= in can_use_umr_rereg_pas()
1765 static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, in umr_rereg_pas() argument
1769 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); in umr_rereg_pas()
1771 struct ib_umem *old_umem = mr->umem; in umr_rereg_pas()
1784 mr->ibmr.pd = pd; in umr_rereg_pas()
1788 mr->access_flags = access_flags; in umr_rereg_pas()
1792 mr->ibmr.iova = iova; in umr_rereg_pas()
1793 mr->ibmr.length = new_umem->length; in umr_rereg_pas()
1794 mr->page_shift = order_base_2(page_size); in umr_rereg_pas()
1795 mr->umem = new_umem; in umr_rereg_pas()
1802 mr->umem = old_umem; in umr_rereg_pas()
1806 atomic_sub(ib_umem_num_pages(old_umem), &dev->mdev->priv.reg_pages); in umr_rereg_pas()
1808 atomic_add(ib_umem_num_pages(new_umem), &dev->mdev->priv.reg_pages); in umr_rereg_pas()
1817 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device); in mlx5_ib_rereg_user_mr()
1821 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM) || mr->data_direct) in mlx5_ib_rereg_user_mr()
1822 return ERR_PTR(-EOPNOTSUPP); in mlx5_ib_rereg_user_mr()
1830 return ERR_PTR(-EOPNOTSUPP); in mlx5_ib_rereg_user_mr()
1833 new_access_flags = mr->access_flags; in mlx5_ib_rereg_user_mr()
1835 new_pd = ib_mr->pd; in mlx5_ib_rereg_user_mr()
1840 /* Fast path for PD/access change */ in mlx5_ib_rereg_user_mr()
1841 if (can_use_umr_rereg_access(dev, mr->access_flags, in mlx5_ib_rereg_user_mr()
1849 /* DM or ODP MR's don't have a normal umem so we can't re-use it */ in mlx5_ib_rereg_user_mr()
1850 if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr)) in mlx5_ib_rereg_user_mr()
1860 umem = mr->umem; in mlx5_ib_rereg_user_mr()
1861 mr->umem = NULL; in mlx5_ib_rereg_user_mr()
1862 atomic_sub(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages); in mlx5_ib_rereg_user_mr()
1864 return create_real_mr(new_pd, umem, mr->ibmr.iova, in mlx5_ib_rereg_user_mr()
1869 * DM doesn't have a PAS list so we can't re-use it, odp/dmabuf does in mlx5_ib_rereg_user_mr()
1872 if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr)) in mlx5_ib_rereg_user_mr()
1876 can_use_umr_rereg_access(dev, mr->access_flags, new_access_flags)) { in mlx5_ib_rereg_user_mr()
1880 new_umem = ib_umem_get(&dev->ib_dev, start, length, in mlx5_ib_rereg_user_mr()
1915 struct device *ddev = &dev->mdev->pdev->dev; in mlx5_alloc_priv_descs()
1920 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0); in mlx5_alloc_priv_descs()
1924 add_size = min_t(int, end - size, add_size); in mlx5_alloc_priv_descs()
1927 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL); in mlx5_alloc_priv_descs()
1928 if (!mr->descs_alloc) in mlx5_alloc_priv_descs()
1929 return -ENOMEM; in mlx5_alloc_priv_descs()
1931 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN); in mlx5_alloc_priv_descs()
1933 mr->desc_map = dma_map_single(ddev, mr->descs, size, DMA_TO_DEVICE); in mlx5_alloc_priv_descs()
1934 if (dma_mapping_error(ddev, mr->desc_map)) { in mlx5_alloc_priv_descs()
1935 ret = -ENOMEM; in mlx5_alloc_priv_descs()
1941 kfree(mr->descs_alloc); in mlx5_alloc_priv_descs()
1949 if (!mr->umem && !mr->data_direct && in mlx5_free_priv_descs()
1950 mr->ibmr.type != IB_MR_TYPE_DM && mr->descs) { in mlx5_free_priv_descs()
1951 struct ib_device *device = mr->ibmr.device; in mlx5_free_priv_descs()
1952 int size = mr->max_descs * mr->desc_size; in mlx5_free_priv_descs()
1955 dma_unmap_single(&dev->mdev->pdev->dev, mr->desc_map, size, in mlx5_free_priv_descs()
1957 kfree(mr->descs_alloc); in mlx5_free_priv_descs()
1958 mr->descs = NULL; in mlx5_free_priv_descs()
1965 struct mlx5_mkey_cache *cache = &dev->cache; in cache_ent_find_and_store()
1969 if (mr->mmkey.cache_ent) { in cache_ent_find_and_store()
1970 spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock); in cache_ent_find_and_store()
1971 mr->mmkey.cache_ent->in_use--; in cache_ent_find_and_store()
1975 mutex_lock(&cache->rb_lock); in cache_ent_find_and_store()
1976 ent = mkey_cache_ent_from_rb_key(dev, mr->mmkey.rb_key); in cache_ent_find_and_store()
1978 if (ent->rb_key.ndescs == mr->mmkey.rb_key.ndescs) { in cache_ent_find_and_store()
1979 if (ent->disabled) { in cache_ent_find_and_store()
1980 mutex_unlock(&cache->rb_lock); in cache_ent_find_and_store()
1981 return -EOPNOTSUPP; in cache_ent_find_and_store()
1983 mr->mmkey.cache_ent = ent; in cache_ent_find_and_store()
1984 spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock); in cache_ent_find_and_store()
1985 mutex_unlock(&cache->rb_lock); in cache_ent_find_and_store()
1990 ent = mlx5r_cache_create_ent_locked(dev, mr->mmkey.rb_key, false); in cache_ent_find_and_store()
1991 mutex_unlock(&cache->rb_lock); in cache_ent_find_and_store()
1995 mr->mmkey.cache_ent = ent; in cache_ent_find_and_store()
1996 spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock); in cache_ent_find_and_store()
1999 ret = push_mkey_locked(mr->mmkey.cache_ent, mr->mmkey.key); in cache_ent_find_and_store()
2000 spin_unlock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock); in cache_ent_find_and_store()
2006 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); in mlx5_ib_revoke_data_direct_mr()
2007 struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem); in mlx5_ib_revoke_data_direct_mr()
2010 lockdep_assert_held(&dev->data_direct_lock); in mlx5_ib_revoke_data_direct_mr()
2011 mr->revoked = true; in mlx5_ib_revoke_data_direct_mr()
2024 lockdep_assert_held(&dev->data_direct_lock); in mlx5_ib_revoke_data_direct_mrs()
2026 list_for_each_entry_safe(mr, next, &dev->data_direct_mr_list, dd_node) { in mlx5_ib_revoke_data_direct_mrs()
2027 list_del(&mr->dd_node); in mlx5_ib_revoke_data_direct_mrs()
2034 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); in mlx5_revoke_mr()
2035 struct mlx5_cache_ent *ent = mr->mmkey.cache_ent; in mlx5_revoke_mr()
2038 !to_ib_umem_dmabuf(mr->umem)->pinned; in mlx5_revoke_mr()
2042 mutex_lock(&to_ib_umem_odp(mr->umem)->umem_mutex); in mlx5_revoke_mr()
2045 dma_resv_lock(to_ib_umem_dmabuf(mr->umem)->attach->dmabuf->resv, NULL); in mlx5_revoke_mr()
2047 if (mr->mmkey.cacheable && !mlx5r_umr_revoke_mr(mr) && !cache_ent_find_and_store(dev, mr)) { in mlx5_revoke_mr()
2048 ent = mr->mmkey.cache_ent; in mlx5_revoke_mr()
2049 /* upon storing to a clean temp entry - schedule its cleanup */ in mlx5_revoke_mr()
2050 spin_lock_irq(&ent->mkeys_queue.lock); in mlx5_revoke_mr()
2051 if (ent->is_tmp && !ent->tmp_cleanup_scheduled) { in mlx5_revoke_mr()
2052 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, in mlx5_revoke_mr()
2054 ent->tmp_cleanup_scheduled = true; in mlx5_revoke_mr()
2056 spin_unlock_irq(&ent->mkeys_queue.lock); in mlx5_revoke_mr()
2061 spin_lock_irq(&ent->mkeys_queue.lock); in mlx5_revoke_mr()
2062 ent->in_use--; in mlx5_revoke_mr()
2063 mr->mmkey.cache_ent = NULL; in mlx5_revoke_mr()
2064 spin_unlock_irq(&ent->mkeys_queue.lock); in mlx5_revoke_mr()
2070 to_ib_umem_odp(mr->umem)->private = NULL; in mlx5_revoke_mr()
2071 mutex_unlock(&to_ib_umem_odp(mr->umem)->umem_mutex); in mlx5_revoke_mr()
2076 to_ib_umem_dmabuf(mr->umem)->private = NULL; in mlx5_revoke_mr()
2077 dma_resv_unlock(to_ib_umem_dmabuf(mr->umem)->attach->dmabuf->resv); in mlx5_revoke_mr()
2086 struct mlx5_ib_dev *dev = to_mdev(ibmr->device); in __mlx5_ib_dereg_mr()
2095 refcount_read(&mr->mmkey.usecount) != 0 && in __mlx5_ib_dereg_mr()
2096 xa_erase(&mr_to_mdev(mr)->odp_mkeys, mlx5_base_mkey(mr->mmkey.key))) in __mlx5_ib_dereg_mr()
2097 mlx5r_deref_wait_odp_mkey(&mr->mmkey); in __mlx5_ib_dereg_mr()
2099 if (ibmr->type == IB_MR_TYPE_INTEGRITY) { in __mlx5_ib_dereg_mr()
2100 xa_cmpxchg(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key), in __mlx5_ib_dereg_mr()
2101 mr->sig, NULL, GFP_KERNEL); in __mlx5_ib_dereg_mr()
2103 if (mr->mtt_mr) { in __mlx5_ib_dereg_mr()
2104 rc = mlx5_ib_dereg_mr(&mr->mtt_mr->ibmr, NULL); in __mlx5_ib_dereg_mr()
2107 mr->mtt_mr = NULL; in __mlx5_ib_dereg_mr()
2109 if (mr->klm_mr) { in __mlx5_ib_dereg_mr()
2110 rc = mlx5_ib_dereg_mr(&mr->klm_mr->ibmr, NULL); in __mlx5_ib_dereg_mr()
2113 mr->klm_mr = NULL; in __mlx5_ib_dereg_mr()
2116 if (mlx5_core_destroy_psv(dev->mdev, in __mlx5_ib_dereg_mr()
2117 mr->sig->psv_memory.psv_idx)) in __mlx5_ib_dereg_mr()
2119 mr->sig->psv_memory.psv_idx); in __mlx5_ib_dereg_mr()
2120 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx)) in __mlx5_ib_dereg_mr()
2122 mr->sig->psv_wire.psv_idx); in __mlx5_ib_dereg_mr()
2123 kfree(mr->sig); in __mlx5_ib_dereg_mr()
2124 mr->sig = NULL; in __mlx5_ib_dereg_mr()
2132 if (mr->umem) { in __mlx5_ib_dereg_mr()
2136 atomic_sub(ib_umem_num_pages(mr->umem), in __mlx5_ib_dereg_mr()
2137 &dev->mdev->priv.reg_pages); in __mlx5_ib_dereg_mr()
2138 ib_umem_release(mr->umem); in __mlx5_ib_dereg_mr()
2143 if (!mr->mmkey.cache_ent) in __mlx5_ib_dereg_mr()
2153 struct mlx5_ib_mr *dd_crossed_mr = mr->dd_crossed_mr; in dereg_crossing_data_direct_mr()
2156 ret = __mlx5_ib_dereg_mr(&mr->ibmr); in dereg_crossing_data_direct_mr()
2160 mutex_lock(&dev->data_direct_lock); in dereg_crossing_data_direct_mr()
2161 if (!dd_crossed_mr->revoked) in dereg_crossing_data_direct_mr()
2162 list_del(&dd_crossed_mr->dd_node); in dereg_crossing_data_direct_mr()
2164 ret = __mlx5_ib_dereg_mr(&dd_crossed_mr->ibmr); in dereg_crossing_data_direct_mr()
2165 mutex_unlock(&dev->data_direct_lock); in dereg_crossing_data_direct_mr()
2172 struct mlx5_ib_dev *dev = to_mdev(ibmr->device); in mlx5_ib_dereg_mr()
2174 if (mr->data_direct) in mlx5_ib_dereg_mr()
2180 static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs, in mlx5_set_umr_free_mkey() argument
2183 struct mlx5_ib_dev *dev = to_mdev(pd->device); in mlx5_set_umr_free_mkey()
2188 /* This is only used from the kernel, so setting the PD is OK. */ in mlx5_set_umr_free_mkey()
2189 set_mkc_access_pd_addr_fields(mkc, IB_ACCESS_RELAXED_ORDERING, 0, pd); in mlx5_set_umr_free_mkey()
2198 MLX5_SET(mkc, mkc, ma_translation_mode, MLX5_CAP_GEN(dev->mdev, ats)); in mlx5_set_umr_free_mkey()
2201 static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, in _mlx5_alloc_mkey_descs() argument
2205 struct mlx5_ib_dev *dev = to_mdev(pd->device); in _mlx5_alloc_mkey_descs()
2208 mr->access_mode = access_mode; in _mlx5_alloc_mkey_descs()
2209 mr->desc_size = desc_size; in _mlx5_alloc_mkey_descs()
2210 mr->max_descs = ndescs; in _mlx5_alloc_mkey_descs()
2212 err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size); in _mlx5_alloc_mkey_descs()
2216 mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift); in _mlx5_alloc_mkey_descs()
2218 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); in _mlx5_alloc_mkey_descs()
2222 mr->mmkey.type = MLX5_MKEY_MR; in _mlx5_alloc_mkey_descs()
2223 mr->ibmr.lkey = mr->mmkey.key; in _mlx5_alloc_mkey_descs()
2224 mr->ibmr.rkey = mr->mmkey.key; in _mlx5_alloc_mkey_descs()
2233 static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd, in mlx5_ib_alloc_pi_mr() argument
2246 return ERR_PTR(-ENOMEM); in mlx5_ib_alloc_pi_mr()
2248 mr->ibmr.pd = pd; in mlx5_ib_alloc_pi_mr()
2249 mr->ibmr.device = pd->device; in mlx5_ib_alloc_pi_mr()
2253 err = -ENOMEM; in mlx5_ib_alloc_pi_mr()
2260 err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift, in mlx5_ib_alloc_pi_mr()
2265 mr->umem = NULL; in mlx5_ib_alloc_pi_mr()
2277 static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, in mlx5_alloc_mem_reg_descs() argument
2280 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt), in mlx5_alloc_mem_reg_descs()
2285 static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, in mlx5_alloc_sg_gaps_descs() argument
2288 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm), in mlx5_alloc_sg_gaps_descs()
2292 static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, in mlx5_alloc_integrity_descs() argument
2296 struct mlx5_ib_dev *dev = to_mdev(pd->device); in mlx5_alloc_integrity_descs()
2301 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); in mlx5_alloc_integrity_descs()
2302 if (!mr->sig) in mlx5_alloc_integrity_descs()
2303 return -ENOMEM; in mlx5_alloc_integrity_descs()
2306 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index); in mlx5_alloc_integrity_descs()
2310 mr->sig->psv_memory.psv_idx = psv_index[0]; in mlx5_alloc_integrity_descs()
2311 mr->sig->psv_wire.psv_idx = psv_index[1]; in mlx5_alloc_integrity_descs()
2313 mr->sig->sig_status_checked = true; in mlx5_alloc_integrity_descs()
2314 mr->sig->sig_err_exists = false; in mlx5_alloc_integrity_descs()
2316 ++mr->sig->sigerr_count; in mlx5_alloc_integrity_descs()
2317 mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, in mlx5_alloc_integrity_descs()
2320 if (IS_ERR(mr->klm_mr)) { in mlx5_alloc_integrity_descs()
2321 err = PTR_ERR(mr->klm_mr); in mlx5_alloc_integrity_descs()
2324 mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, in mlx5_alloc_integrity_descs()
2327 if (IS_ERR(mr->mtt_mr)) { in mlx5_alloc_integrity_descs()
2328 err = PTR_ERR(mr->mtt_mr); in mlx5_alloc_integrity_descs()
2337 err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0, in mlx5_alloc_integrity_descs()
2342 err = xa_err(xa_store(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key), in mlx5_alloc_integrity_descs()
2343 mr->sig, GFP_KERNEL)); in mlx5_alloc_integrity_descs()
2352 mlx5_ib_dereg_mr(&mr->mtt_mr->ibmr, NULL); in mlx5_alloc_integrity_descs()
2353 mr->mtt_mr = NULL; in mlx5_alloc_integrity_descs()
2355 mlx5_ib_dereg_mr(&mr->klm_mr->ibmr, NULL); in mlx5_alloc_integrity_descs()
2356 mr->klm_mr = NULL; in mlx5_alloc_integrity_descs()
2358 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx)) in mlx5_alloc_integrity_descs()
2360 mr->sig->psv_memory.psv_idx); in mlx5_alloc_integrity_descs()
2361 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx)) in mlx5_alloc_integrity_descs()
2363 mr->sig->psv_wire.psv_idx); in mlx5_alloc_integrity_descs()
2365 kfree(mr->sig); in mlx5_alloc_integrity_descs()
2370 static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, in __mlx5_ib_alloc_mr() argument
2374 struct mlx5_ib_dev *dev = to_mdev(pd->device); in __mlx5_ib_alloc_mr()
2383 return ERR_PTR(-ENOMEM); in __mlx5_ib_alloc_mr()
2387 err = -ENOMEM; in __mlx5_ib_alloc_mr()
2391 mr->ibmr.device = pd->device; in __mlx5_ib_alloc_mr()
2392 mr->umem = NULL; in __mlx5_ib_alloc_mr()
2396 err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen); in __mlx5_ib_alloc_mr()
2399 err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen); in __mlx5_ib_alloc_mr()
2402 err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg, in __mlx5_ib_alloc_mr()
2407 err = -EINVAL; in __mlx5_ib_alloc_mr()
2415 return &mr->ibmr; in __mlx5_ib_alloc_mr()
2424 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, in mlx5_ib_alloc_mr() argument
2427 return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0); in mlx5_ib_alloc_mr()
2430 struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, in mlx5_ib_alloc_mr_integrity() argument
2433 return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg, in mlx5_ib_alloc_mr_integrity()
2439 struct mlx5_ib_dev *dev = to_mdev(ibmw->device); in mlx5_ib_alloc_mw()
2452 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req))); in mlx5_ib_alloc_mw()
2457 return -EOPNOTSUPP; in mlx5_ib_alloc_mw()
2459 if (udata->inlen > sizeof(req) && in mlx5_ib_alloc_mw()
2461 udata->inlen - sizeof(req))) in mlx5_ib_alloc_mw()
2462 return -EOPNOTSUPP; in mlx5_ib_alloc_mw()
2468 return -ENOMEM; in mlx5_ib_alloc_mw()
2474 MLX5_SET(mkc, mkc, pd, to_mpd(ibmw->pd)->pdn); in mlx5_ib_alloc_mw()
2478 MLX5_SET(mkc, mkc, en_rinval, !!((ibmw->type == IB_MW_TYPE_2))); in mlx5_ib_alloc_mw()
2481 err = mlx5_ib_create_mkey(dev, &mw->mmkey, in, inlen); in mlx5_ib_alloc_mw()
2485 mw->mmkey.type = MLX5_MKEY_MW; in mlx5_ib_alloc_mw()
2486 ibmw->rkey = mw->mmkey.key; in mlx5_ib_alloc_mw()
2487 mw->mmkey.ndescs = ndescs; in mlx5_ib_alloc_mw()
2490 min(offsetofend(typeof(resp), response_length), udata->outlen); in mlx5_ib_alloc_mw()
2498 err = mlx5r_store_odp_mkey(dev, &mw->mmkey); in mlx5_ib_alloc_mw()
2507 mlx5_core_destroy_mkey(dev->mdev, mw->mmkey.key); in mlx5_ib_alloc_mw()
2515 struct mlx5_ib_dev *dev = to_mdev(mw->device); in mlx5_ib_dealloc_mw()
2519 xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key))) in mlx5_ib_dealloc_mw()
2524 mlx5r_deref_wait_odp_mkey(&mmw->mmkey); in mlx5_ib_dealloc_mw()
2526 return mlx5_core_destroy_mkey(dev->mdev, mmw->mmkey.key); in mlx5_ib_dealloc_mw()
2537 ret = -EINVAL; in mlx5_ib_check_mr_status()
2541 mr_status->fail_status = 0; in mlx5_ib_check_mr_status()
2543 if (!mmr->sig) { in mlx5_ib_check_mr_status()
2544 ret = -EINVAL; in mlx5_ib_check_mr_status()
2545 pr_err("signature status check requested on a non-signature enabled MR\n"); in mlx5_ib_check_mr_status()
2549 mmr->sig->sig_status_checked = true; in mlx5_ib_check_mr_status()
2550 if (!mmr->sig->sig_err_exists) in mlx5_ib_check_mr_status()
2553 if (ibmr->lkey == mmr->sig->err_item.key) in mlx5_ib_check_mr_status()
2554 memcpy(&mr_status->sig_err, &mmr->sig->err_item, in mlx5_ib_check_mr_status()
2555 sizeof(mr_status->sig_err)); in mlx5_ib_check_mr_status()
2557 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD; in mlx5_ib_check_mr_status()
2558 mr_status->sig_err.sig_err_offset = 0; in mlx5_ib_check_mr_status()
2559 mr_status->sig_err.key = mmr->sig->err_item.key; in mlx5_ib_check_mr_status()
2562 mmr->sig->sig_err_exists = false; in mlx5_ib_check_mr_status()
2563 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; in mlx5_ib_check_mr_status()
2580 mr->meta_length = 0; in mlx5_ib_map_pa_mr_sg_pi()
2583 mr->mmkey.ndescs = 1; in mlx5_ib_map_pa_mr_sg_pi()
2586 mr->data_length = sg_dma_len(data_sg) - sg_offset; in mlx5_ib_map_pa_mr_sg_pi()
2587 mr->data_iova = sg_dma_address(data_sg) + sg_offset; in mlx5_ib_map_pa_mr_sg_pi()
2590 mr->meta_ndescs = 1; in mlx5_ib_map_pa_mr_sg_pi()
2595 mr->meta_length = sg_dma_len(meta_sg) - sg_offset; in mlx5_ib_map_pa_mr_sg_pi()
2596 mr->pi_iova = sg_dma_address(meta_sg) + sg_offset; in mlx5_ib_map_pa_mr_sg_pi()
2598 ibmr->length = mr->data_length + mr->meta_length; in mlx5_ib_map_pa_mr_sg_pi()
2614 struct mlx5_klm *klms = mr->descs; in mlx5_ib_sg_to_klms()
2616 u32 lkey = mr->ibmr.pd->local_dma_lkey; in mlx5_ib_sg_to_klms()
2619 mr->ibmr.iova = sg_dma_address(sg) + sg_offset; in mlx5_ib_sg_to_klms()
2620 mr->ibmr.length = 0; in mlx5_ib_sg_to_klms()
2623 if (unlikely(i >= mr->max_descs)) in mlx5_ib_sg_to_klms()
2626 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset); in mlx5_ib_sg_to_klms()
2628 mr->ibmr.length += sg_dma_len(sg) - sg_offset; in mlx5_ib_sg_to_klms()
2636 mr->mmkey.ndescs = i; in mlx5_ib_sg_to_klms()
2637 mr->data_length = mr->ibmr.length; in mlx5_ib_sg_to_klms()
2643 if (unlikely(i + j >= mr->max_descs)) in mlx5_ib_sg_to_klms()
2647 klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) - in mlx5_ib_sg_to_klms()
2650 mr->ibmr.length += sg_dma_len(sg) - sg_offset; in mlx5_ib_sg_to_klms()
2657 mr->meta_ndescs = j; in mlx5_ib_sg_to_klms()
2658 mr->meta_length = mr->ibmr.length - mr->data_length; in mlx5_ib_sg_to_klms()
2669 if (unlikely(mr->mmkey.ndescs == mr->max_descs)) in mlx5_set_page()
2670 return -ENOMEM; in mlx5_set_page()
2672 descs = mr->descs; in mlx5_set_page()
2673 descs[mr->mmkey.ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); in mlx5_set_page()
2683 if (unlikely(mr->mmkey.ndescs + mr->meta_ndescs == mr->max_descs)) in mlx5_set_page_pi()
2684 return -ENOMEM; in mlx5_set_page_pi()
2686 descs = mr->descs; in mlx5_set_page_pi()
2687 descs[mr->mmkey.ndescs + mr->meta_ndescs++] = in mlx5_set_page_pi()
2700 struct mlx5_ib_mr *pi_mr = mr->mtt_mr; in mlx5_ib_map_mtt_mr_sg_pi()
2703 pi_mr->mmkey.ndescs = 0; in mlx5_ib_map_mtt_mr_sg_pi()
2704 pi_mr->meta_ndescs = 0; in mlx5_ib_map_mtt_mr_sg_pi()
2705 pi_mr->meta_length = 0; in mlx5_ib_map_mtt_mr_sg_pi()
2707 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, in mlx5_ib_map_mtt_mr_sg_pi()
2708 pi_mr->desc_size * pi_mr->max_descs, in mlx5_ib_map_mtt_mr_sg_pi()
2711 pi_mr->ibmr.page_size = ibmr->page_size; in mlx5_ib_map_mtt_mr_sg_pi()
2712 n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset, in mlx5_ib_map_mtt_mr_sg_pi()
2717 pi_mr->data_iova = pi_mr->ibmr.iova; in mlx5_ib_map_mtt_mr_sg_pi()
2718 pi_mr->data_length = pi_mr->ibmr.length; in mlx5_ib_map_mtt_mr_sg_pi()
2719 pi_mr->ibmr.length = pi_mr->data_length; in mlx5_ib_map_mtt_mr_sg_pi()
2720 ibmr->length = pi_mr->data_length; in mlx5_ib_map_mtt_mr_sg_pi()
2723 u64 page_mask = ~((u64)ibmr->page_size - 1); in mlx5_ib_map_mtt_mr_sg_pi()
2724 u64 iova = pi_mr->data_iova; in mlx5_ib_map_mtt_mr_sg_pi()
2726 n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents, in mlx5_ib_map_mtt_mr_sg_pi()
2729 pi_mr->meta_length = pi_mr->ibmr.length; in mlx5_ib_map_mtt_mr_sg_pi()
2736 pi_mr->pi_iova = (iova & page_mask) + in mlx5_ib_map_mtt_mr_sg_pi()
2737 pi_mr->mmkey.ndescs * ibmr->page_size + in mlx5_ib_map_mtt_mr_sg_pi()
2738 (pi_mr->ibmr.iova & ~page_mask); in mlx5_ib_map_mtt_mr_sg_pi()
2746 pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova; in mlx5_ib_map_mtt_mr_sg_pi()
2747 pi_mr->ibmr.iova = iova; in mlx5_ib_map_mtt_mr_sg_pi()
2748 ibmr->length += pi_mr->meta_length; in mlx5_ib_map_mtt_mr_sg_pi()
2751 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, in mlx5_ib_map_mtt_mr_sg_pi()
2752 pi_mr->desc_size * pi_mr->max_descs, in mlx5_ib_map_mtt_mr_sg_pi()
2765 struct mlx5_ib_mr *pi_mr = mr->klm_mr; in mlx5_ib_map_klm_mr_sg_pi()
2768 pi_mr->mmkey.ndescs = 0; in mlx5_ib_map_klm_mr_sg_pi()
2769 pi_mr->meta_ndescs = 0; in mlx5_ib_map_klm_mr_sg_pi()
2770 pi_mr->meta_length = 0; in mlx5_ib_map_klm_mr_sg_pi()
2772 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, in mlx5_ib_map_klm_mr_sg_pi()
2773 pi_mr->desc_size * pi_mr->max_descs, in mlx5_ib_map_klm_mr_sg_pi()
2779 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, in mlx5_ib_map_klm_mr_sg_pi()
2780 pi_mr->desc_size * pi_mr->max_descs, in mlx5_ib_map_klm_mr_sg_pi()
2783 /* This is zero-based memory region */ in mlx5_ib_map_klm_mr_sg_pi()
2784 pi_mr->data_iova = 0; in mlx5_ib_map_klm_mr_sg_pi()
2785 pi_mr->ibmr.iova = 0; in mlx5_ib_map_klm_mr_sg_pi()
2786 pi_mr->pi_iova = pi_mr->data_length; in mlx5_ib_map_klm_mr_sg_pi()
2787 ibmr->length = pi_mr->ibmr.length; in mlx5_ib_map_klm_mr_sg_pi()
2801 WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY); in mlx5_ib_map_mr_sg_pi()
2803 mr->mmkey.ndescs = 0; in mlx5_ib_map_mr_sg_pi()
2804 mr->data_length = 0; in mlx5_ib_map_mr_sg_pi()
2805 mr->data_iova = 0; in mlx5_ib_map_mr_sg_pi()
2806 mr->meta_ndescs = 0; in mlx5_ib_map_mr_sg_pi()
2807 mr->pi_iova = 0; in mlx5_ib_map_mr_sg_pi()
2827 pi_mr = mr->mtt_mr; in mlx5_ib_map_mr_sg_pi()
2834 pi_mr = mr->klm_mr; in mlx5_ib_map_mr_sg_pi()
2839 return -ENOMEM; in mlx5_ib_map_mr_sg_pi()
2842 /* This is zero-based memory region */ in mlx5_ib_map_mr_sg_pi()
2843 ibmr->iova = 0; in mlx5_ib_map_mr_sg_pi()
2844 mr->pi_mr = pi_mr; in mlx5_ib_map_mr_sg_pi()
2846 ibmr->sig_attrs->meta_length = pi_mr->meta_length; in mlx5_ib_map_mr_sg_pi()
2848 ibmr->sig_attrs->meta_length = mr->meta_length; in mlx5_ib_map_mr_sg_pi()
2859 mr->mmkey.ndescs = 0; in mlx5_ib_map_mr_sg()
2861 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map, in mlx5_ib_map_mr_sg()
2862 mr->desc_size * mr->max_descs, in mlx5_ib_map_mr_sg()
2865 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) in mlx5_ib_map_mr_sg()
2872 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map, in mlx5_ib_map_mr_sg()
2873 mr->desc_size * mr->max_descs, in mlx5_ib_map_mr_sg()