Lines Matching +full:elm +full:- +full:id
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
18 rc = iopt_unmap_all(&ioas->iopt, NULL); in iommufd_ioas_destroy()
19 WARN_ON(rc && rc != -ENOENT); in iommufd_ioas_destroy()
20 iopt_destroy_table(&ioas->iopt); in iommufd_ioas_destroy()
21 mutex_destroy(&ioas->mutex); in iommufd_ioas_destroy()
32 iopt_init_table(&ioas->iopt); in iommufd_ioas_alloc()
33 INIT_LIST_HEAD(&ioas->hwpt_list); in iommufd_ioas_alloc()
34 mutex_init(&ioas->mutex); in iommufd_ioas_alloc()
40 struct iommu_ioas_alloc *cmd = ucmd->cmd; in iommufd_ioas_alloc_ioctl()
44 if (cmd->flags) in iommufd_ioas_alloc_ioctl()
45 return -EOPNOTSUPP; in iommufd_ioas_alloc_ioctl()
47 ioas = iommufd_ioas_alloc(ucmd->ictx); in iommufd_ioas_alloc_ioctl()
51 cmd->out_ioas_id = ioas->obj.id; in iommufd_ioas_alloc_ioctl()
56 down_read(&ucmd->ictx->ioas_creation_lock); in iommufd_ioas_alloc_ioctl()
57 iommufd_object_finalize(ucmd->ictx, &ioas->obj); in iommufd_ioas_alloc_ioctl()
58 up_read(&ucmd->ictx->ioas_creation_lock); in iommufd_ioas_alloc_ioctl()
62 iommufd_object_abort_and_destroy(ucmd->ictx, &ioas->obj); in iommufd_ioas_alloc_ioctl()
69 struct iommu_ioas_iova_ranges *cmd = ucmd->cmd; in iommufd_ioas_iova_ranges()
75 if (cmd->__reserved) in iommufd_ioas_iova_ranges()
76 return -EOPNOTSUPP; in iommufd_ioas_iova_ranges()
78 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); in iommufd_ioas_iova_ranges()
82 down_read(&ioas->iopt.iova_rwsem); in iommufd_ioas_iova_ranges()
83 max_iovas = cmd->num_iovas; in iommufd_ioas_iova_ranges()
84 ranges = u64_to_user_ptr(cmd->allowed_iovas); in iommufd_ioas_iova_ranges()
85 cmd->num_iovas = 0; in iommufd_ioas_iova_ranges()
86 cmd->out_iova_alignment = ioas->iopt.iova_alignment; in iommufd_ioas_iova_ranges()
87 interval_tree_for_each_span(&span, &ioas->iopt.reserved_itree, 0, in iommufd_ioas_iova_ranges()
91 if (cmd->num_iovas < max_iovas) { in iommufd_ioas_iova_ranges()
92 struct iommu_iova_range elm = { in iommufd_ioas_iova_ranges() local
97 if (copy_to_user(&ranges[cmd->num_iovas], &elm, in iommufd_ioas_iova_ranges()
98 sizeof(elm))) { in iommufd_ioas_iova_ranges()
99 rc = -EFAULT; in iommufd_ioas_iova_ranges()
103 cmd->num_iovas++; in iommufd_ioas_iova_ranges()
108 if (cmd->num_iovas > max_iovas) in iommufd_ioas_iova_ranges()
109 rc = -EMSGSIZE; in iommufd_ioas_iova_ranges()
111 up_read(&ioas->iopt.iova_rwsem); in iommufd_ioas_iova_ranges()
112 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_iova_ranges()
127 return -EFAULT; in iommufd_ioas_load_iovas()
130 return -EINVAL; in iommufd_ioas_load_iovas()
133 return -EINVAL; in iommufd_ioas_load_iovas()
137 return -ENOMEM; in iommufd_ioas_load_iovas()
138 allowed->node.start = range.start; in iommufd_ioas_load_iovas()
139 allowed->node.last = range.last; in iommufd_ioas_load_iovas()
141 interval_tree_insert(&allowed->node, itree); in iommufd_ioas_load_iovas()
148 struct iommu_ioas_allow_iovas *cmd = ucmd->cmd; in iommufd_ioas_allow_iovas()
155 if (cmd->__reserved) in iommufd_ioas_allow_iovas()
156 return -EOPNOTSUPP; in iommufd_ioas_allow_iovas()
158 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); in iommufd_ioas_allow_iovas()
161 iopt = &ioas->iopt; in iommufd_ioas_allow_iovas()
164 u64_to_user_ptr(cmd->allowed_iovas), in iommufd_ioas_allow_iovas()
165 cmd->num_iovas); in iommufd_ioas_allow_iovas()
182 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_allow_iovas()
206 struct iommu_ioas_map_file *cmd = ucmd->cmd; in iommufd_ioas_map_file()
207 unsigned long iova = cmd->iova; in iommufd_ioas_map_file()
213 if (cmd->flags & in iommufd_ioas_map_file()
216 return -EOPNOTSUPP; in iommufd_ioas_map_file()
218 if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) in iommufd_ioas_map_file()
219 return -EOVERFLOW; in iommufd_ioas_map_file()
221 if (!(cmd->flags & in iommufd_ioas_map_file()
223 return -EINVAL; in iommufd_ioas_map_file()
225 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); in iommufd_ioas_map_file()
229 if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA)) in iommufd_ioas_map_file()
232 file = fget(cmd->fd); in iommufd_ioas_map_file()
234 return -EBADF; in iommufd_ioas_map_file()
236 rc = iopt_map_file_pages(ucmd->ictx, &ioas->iopt, &iova, file, in iommufd_ioas_map_file()
237 cmd->start, cmd->length, in iommufd_ioas_map_file()
238 conv_iommu_prot(cmd->flags), flags); in iommufd_ioas_map_file()
242 cmd->iova = iova; in iommufd_ioas_map_file()
245 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_map_file()
252 struct iommu_ioas_map *cmd = ucmd->cmd; in iommufd_ioas_map()
253 unsigned long iova = cmd->iova; in iommufd_ioas_map()
258 if ((cmd->flags & in iommufd_ioas_map()
261 cmd->__reserved) in iommufd_ioas_map()
262 return -EOPNOTSUPP; in iommufd_ioas_map()
263 if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) in iommufd_ioas_map()
264 return -EOVERFLOW; in iommufd_ioas_map()
266 if (!(cmd->flags & in iommufd_ioas_map()
268 return -EINVAL; in iommufd_ioas_map()
270 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); in iommufd_ioas_map()
274 if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA)) in iommufd_ioas_map()
276 rc = iopt_map_user_pages(ucmd->ictx, &ioas->iopt, &iova, in iommufd_ioas_map()
277 u64_to_user_ptr(cmd->user_va), cmd->length, in iommufd_ioas_map()
278 conv_iommu_prot(cmd->flags), flags); in iommufd_ioas_map()
282 cmd->iova = iova; in iommufd_ioas_map()
285 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_map()
291 struct iommu_ioas_copy *cmd = ucmd->cmd; in iommufd_ioas_copy()
299 iommufd_test_syz_conv_iova_id(ucmd, cmd->src_ioas_id, &cmd->src_iova, in iommufd_ioas_copy()
300 &cmd->flags); in iommufd_ioas_copy()
302 if ((cmd->flags & in iommufd_ioas_copy()
305 return -EOPNOTSUPP; in iommufd_ioas_copy()
306 if (cmd->length >= ULONG_MAX || cmd->src_iova >= ULONG_MAX || in iommufd_ioas_copy()
307 cmd->dst_iova >= ULONG_MAX) in iommufd_ioas_copy()
308 return -EOVERFLOW; in iommufd_ioas_copy()
310 if (!(cmd->flags & in iommufd_ioas_copy()
312 return -EINVAL; in iommufd_ioas_copy()
314 src_ioas = iommufd_get_ioas(ucmd->ictx, cmd->src_ioas_id); in iommufd_ioas_copy()
317 rc = iopt_get_pages(&src_ioas->iopt, cmd->src_iova, cmd->length, in iommufd_ioas_copy()
319 iommufd_put_object(ucmd->ictx, &src_ioas->obj); in iommufd_ioas_copy()
323 dst_ioas = iommufd_get_ioas(ucmd->ictx, cmd->dst_ioas_id); in iommufd_ioas_copy()
329 if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA)) in iommufd_ioas_copy()
331 iova = cmd->dst_iova; in iommufd_ioas_copy()
332 rc = iopt_map_pages(&dst_ioas->iopt, &pages_list, cmd->length, &iova, in iommufd_ioas_copy()
333 conv_iommu_prot(cmd->flags), flags); in iommufd_ioas_copy()
337 cmd->dst_iova = iova; in iommufd_ioas_copy()
340 iommufd_put_object(ucmd->ictx, &dst_ioas->obj); in iommufd_ioas_copy()
348 struct iommu_ioas_unmap *cmd = ucmd->cmd; in iommufd_ioas_unmap()
353 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); in iommufd_ioas_unmap()
357 if (cmd->iova == 0 && cmd->length == U64_MAX) { in iommufd_ioas_unmap()
358 rc = iopt_unmap_all(&ioas->iopt, &unmapped); in iommufd_ioas_unmap()
362 if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) { in iommufd_ioas_unmap()
363 rc = -EOVERFLOW; in iommufd_ioas_unmap()
366 rc = iopt_unmap_iova(&ioas->iopt, cmd->iova, cmd->length, in iommufd_ioas_unmap()
372 cmd->length = unmapped; in iommufd_ioas_unmap()
376 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_unmap()
387 up_write(&ioas->iopt.iova_rwsem); in iommufd_release_all_iova_rwsem()
388 refcount_dec(&ioas->obj.users); in iommufd_release_all_iova_rwsem()
390 up_write(&ictx->ioas_creation_lock); in iommufd_release_all_iova_rwsem()
403 * pages->source_mm, which is a performance path for mdev, we just in iommufd_take_all_iova_rwsem()
405 * pages->source_*. Due to copies we can't know which IOAS could read in iommufd_take_all_iova_rwsem()
407 * locks are nested and they are uniformly taken in ID order. in iommufd_take_all_iova_rwsem()
413 down_write(&ictx->ioas_creation_lock); in iommufd_take_all_iova_rwsem()
414 xa_lock(&ictx->objects); in iommufd_take_all_iova_rwsem()
415 xa_for_each(&ictx->objects, index, obj) { in iommufd_take_all_iova_rwsem()
418 if (!obj || obj->type != IOMMUFD_OBJ_IOAS) in iommufd_take_all_iova_rwsem()
421 if (!refcount_inc_not_zero(&obj->users)) in iommufd_take_all_iova_rwsem()
424 xa_unlock(&ictx->objects); in iommufd_take_all_iova_rwsem()
427 down_write_nest_lock(&ioas->iopt.iova_rwsem, in iommufd_take_all_iova_rwsem()
428 &ictx->ioas_creation_lock); in iommufd_take_all_iova_rwsem()
436 xa_lock(&ictx->objects); in iommufd_take_all_iova_rwsem()
438 xa_unlock(&ictx->objects); in iommufd_take_all_iova_rwsem()
444 switch (pages->account_mode) { in need_charge_update()
448 return pages->source_mm != current->mm; in need_charge_update()
452 * in mm->pinned_vm. in need_charge_update()
454 return (pages->source_user != current_user()) || in need_charge_update()
455 (pages->source_mm != current->mm); in need_charge_update()
463 .source_mm = current->mm, in charge_current()
464 .source_task = current->group_leader, in charge_current()
485 account_mode--; in charge_current()
497 struct task_struct *old_task = pages->source_task; in change_mm()
498 struct user_struct *old_user = pages->source_user; in change_mm()
499 struct mm_struct *old_mm = pages->source_mm; in change_mm()
501 pages->source_mm = current->mm; in change_mm()
502 mmgrab(pages->source_mm); in change_mm()
505 pages->source_task = current->group_leader; in change_mm()
506 get_task_struct(pages->source_task); in change_mm()
509 pages->source_user = get_uid(current_user()); in change_mm()
515 for (_area = iopt_area_iter_first(&_ioas->iopt, 0, ULONG_MAX); \
521 struct iommu_ioas_change_process *cmd = ucmd->cmd; in iommufd_ioas_change_process()
522 struct iommufd_ctx *ictx = ucmd->ictx; in iommufd_ioas_change_process()
531 if (cmd->__reserved) in iommufd_ioas_change_process()
532 return -EOPNOTSUPP; in iommufd_ioas_change_process()
540 if (area->pages->type != IOPT_ADDRESS_FILE) { in iommufd_ioas_change_process()
541 rc = -EINVAL; in iommufd_ioas_change_process()
553 pages = area->pages; in iommufd_ioas_change_process()
556 all_npinned[pages->account_mode] += pages->last_npinned; in iommufd_ioas_change_process()
557 pages->last_npinned = 0; in iommufd_ioas_change_process()
566 area->pages->last_npinned = area->pages->npinned; in iommufd_ioas_change_process()
571 pages = area->pages; in iommufd_ioas_change_process()
575 int r = iopt_pages_update_pinned(pages, pages->npinned, in iommufd_ioas_change_process()
592 if (cmd->object_id) in iommufd_option_rlimit_mode()
593 return -EOPNOTSUPP; in iommufd_option_rlimit_mode()
595 if (cmd->op == IOMMU_OPTION_OP_GET) { in iommufd_option_rlimit_mode()
596 cmd->val64 = ictx->account_mode == IOPT_PAGES_ACCOUNT_MM; in iommufd_option_rlimit_mode()
599 if (cmd->op == IOMMU_OPTION_OP_SET) { in iommufd_option_rlimit_mode()
603 return -EPERM; in iommufd_option_rlimit_mode()
605 xa_lock(&ictx->objects); in iommufd_option_rlimit_mode()
606 if (!xa_empty(&ictx->objects)) { in iommufd_option_rlimit_mode()
607 rc = -EBUSY; in iommufd_option_rlimit_mode()
609 if (cmd->val64 == 0) in iommufd_option_rlimit_mode()
610 ictx->account_mode = IOPT_PAGES_ACCOUNT_USER; in iommufd_option_rlimit_mode()
611 else if (cmd->val64 == 1) in iommufd_option_rlimit_mode()
612 ictx->account_mode = IOPT_PAGES_ACCOUNT_MM; in iommufd_option_rlimit_mode()
614 rc = -EINVAL; in iommufd_option_rlimit_mode()
616 xa_unlock(&ictx->objects); in iommufd_option_rlimit_mode()
620 return -EOPNOTSUPP; in iommufd_option_rlimit_mode()
626 if (cmd->op == IOMMU_OPTION_OP_GET) { in iommufd_ioas_option_huge_pages()
627 cmd->val64 = !ioas->iopt.disable_large_pages; in iommufd_ioas_option_huge_pages()
630 if (cmd->op == IOMMU_OPTION_OP_SET) { in iommufd_ioas_option_huge_pages()
631 if (cmd->val64 == 0) in iommufd_ioas_option_huge_pages()
632 return iopt_disable_large_pages(&ioas->iopt); in iommufd_ioas_option_huge_pages()
633 if (cmd->val64 == 1) { in iommufd_ioas_option_huge_pages()
634 iopt_enable_large_pages(&ioas->iopt); in iommufd_ioas_option_huge_pages()
637 return -EINVAL; in iommufd_ioas_option_huge_pages()
639 return -EOPNOTSUPP; in iommufd_ioas_option_huge_pages()
644 struct iommu_option *cmd = ucmd->cmd; in iommufd_ioas_option()
648 if (cmd->__reserved) in iommufd_ioas_option()
649 return -EOPNOTSUPP; in iommufd_ioas_option()
651 ioas = iommufd_get_ioas(ucmd->ictx, cmd->object_id); in iommufd_ioas_option()
655 switch (cmd->option_id) { in iommufd_ioas_option()
660 rc = -EOPNOTSUPP; in iommufd_ioas_option()
663 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_option()