Lines Matching +full:access +full:- +full:granularity
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
3 * Copyright 2020-2021 Advanced Micro Devices, Inc.
84 * svm_range_unlink - unlink svm_range from lists and interval tree
90 * Context: The caller must hold svms->lock
94 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, in svm_range_unlink()
95 prange, prange->start, prange->last); in svm_range_unlink()
97 if (prange->svm_bo) { in svm_range_unlink()
98 spin_lock(&prange->svm_bo->list_lock); in svm_range_unlink()
99 list_del(&prange->svm_bo_list); in svm_range_unlink()
100 spin_unlock(&prange->svm_bo->list_lock); in svm_range_unlink()
103 list_del(&prange->list); in svm_range_unlink()
104 if (prange->it_node.start != 0 && prange->it_node.last != 0) in svm_range_unlink()
105 interval_tree_remove(&prange->it_node, &prange->svms->objects); in svm_range_unlink()
111 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, in svm_range_add_notifier_locked()
112 prange, prange->start, prange->last); in svm_range_add_notifier_locked()
114 mmu_interval_notifier_insert_locked(&prange->notifier, mm, in svm_range_add_notifier_locked()
115 prange->start << PAGE_SHIFT, in svm_range_add_notifier_locked()
116 prange->npages << PAGE_SHIFT, in svm_range_add_notifier_locked()
121 * svm_range_add_to_svms - add svm range to svms
126 * Context: The caller must hold svms->lock
130 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, in svm_range_add_to_svms()
131 prange, prange->start, prange->last); in svm_range_add_to_svms()
133 list_move_tail(&prange->list, &prange->svms->list); in svm_range_add_to_svms()
134 prange->it_node.start = prange->start; in svm_range_add_to_svms()
135 prange->it_node.last = prange->last; in svm_range_add_to_svms()
136 interval_tree_insert(&prange->it_node, &prange->svms->objects); in svm_range_add_to_svms()
142 prange->svms, prange, in svm_range_remove_notifier()
143 prange->notifier.interval_tree.start >> PAGE_SHIFT, in svm_range_remove_notifier()
144 prange->notifier.interval_tree.last >> PAGE_SHIFT); in svm_range_remove_notifier()
146 if (prange->notifier.interval_tree.start != 0 && in svm_range_remove_notifier()
147 prange->notifier.interval_tree.last != 0) in svm_range_remove_notifier()
148 mmu_interval_notifier_remove(&prange->notifier); in svm_range_remove_notifier()
164 dma_addr_t *addr = prange->dma_addr[gpuidx]; in svm_range_dma_map_dev()
165 struct device *dev = adev->dev; in svm_range_dma_map_dev()
170 addr = kvcalloc(prange->npages, sizeof(*addr), GFP_KERNEL); in svm_range_dma_map_dev()
172 return -ENOMEM; in svm_range_dma_map_dev()
173 prange->dma_addr[gpuidx] = addr; in svm_range_dma_map_dev()
183 struct amdgpu_device *bo_adev = prange->svm_bo->node->adev; in svm_range_dma_map_dev()
186 bo_adev->vm_manager.vram_base_offset - in svm_range_dma_map_dev()
187 bo_adev->kfd.pgmap.range.start; in svm_range_dma_map_dev()
214 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_dma_map()
223 return -EINVAL; in svm_range_dma_map()
226 r = svm_range_dma_map_dev(pdd->dev->adev, prange, offset, npages, in svm_range_dma_map()
261 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_dma_unmap()
264 dma_addr = prange->dma_addr[gpuidx]; in svm_range_dma_unmap()
273 dev = &pdd->dev->adev->pdev->dev; in svm_range_dma_unmap()
275 svm_range_dma_unmap_dev(dev, dma_addr, 0, prange->npages); in svm_range_dma_unmap()
281 uint64_t size = (prange->last - prange->start + 1) << PAGE_SHIFT; in svm_range_free()
282 struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms); in svm_range_free()
285 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange, in svm_range_free()
286 prange->start, prange->last); in svm_range_free()
292 if (do_unmap && !p->xnack_enabled) { in svm_range_free()
300 if (prange->dma_addr[gpuidx]) { in svm_range_free()
301 kvfree(prange->dma_addr[gpuidx]); in svm_range_free()
302 prange->dma_addr[gpuidx] = NULL; in svm_range_free()
306 mutex_destroy(&prange->lock); in svm_range_free()
307 mutex_destroy(&prange->migrate_mutex); in svm_range_free()
313 int32_t *prefetch_loc, uint8_t *granularity, in svm_range_set_default_attributes() argument
318 *granularity = svms->default_granularity; in svm_range_set_default_attributes()
327 uint64_t size = last - start + 1; in svm_range_new()
336 if (!p->xnack_enabled && update_mem_usage && in svm_range_new()
343 prange->npages = size; in svm_range_new()
344 prange->svms = svms; in svm_range_new()
345 prange->start = start; in svm_range_new()
346 prange->last = last; in svm_range_new()
347 INIT_LIST_HEAD(&prange->list); in svm_range_new()
348 INIT_LIST_HEAD(&prange->update_list); in svm_range_new()
349 INIT_LIST_HEAD(&prange->svm_bo_list); in svm_range_new()
350 INIT_LIST_HEAD(&prange->deferred_list); in svm_range_new()
351 INIT_LIST_HEAD(&prange->child_list); in svm_range_new()
352 atomic_set(&prange->invalid, 0); in svm_range_new()
353 prange->validate_timestamp = 0; in svm_range_new()
354 prange->vram_pages = 0; in svm_range_new()
355 mutex_init(&prange->migrate_mutex); in svm_range_new()
356 mutex_init(&prange->lock); in svm_range_new()
358 if (p->xnack_enabled) in svm_range_new()
359 bitmap_copy(prange->bitmap_access, svms->bitmap_supported, in svm_range_new()
362 svm_range_set_default_attributes(svms, &prange->preferred_loc, in svm_range_new()
363 &prange->prefetch_loc, in svm_range_new()
364 &prange->granularity, &prange->flags); in svm_range_new()
373 if (!svm_bo || !kref_get_unless_zero(&svm_bo->kref)) in svm_bo_ref_unless_zero()
386 spin_lock(&svm_bo->list_lock); in svm_range_bo_release()
387 while (!list_empty(&svm_bo->range_list)) { in svm_range_bo_release()
389 list_first_entry(&svm_bo->range_list, in svm_range_bo_release()
394 list_del_init(&prange->svm_bo_list); in svm_range_bo_release()
395 spin_unlock(&svm_bo->list_lock); in svm_range_bo_release()
397 pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, in svm_range_bo_release()
398 prange->start, prange->last); in svm_range_bo_release()
399 mutex_lock(&prange->lock); in svm_range_bo_release()
400 prange->svm_bo = NULL; in svm_range_bo_release()
402 WARN_ONCE(prange->actual_loc, "prange should not hold vram page"); in svm_range_bo_release()
403 mutex_unlock(&prange->lock); in svm_range_bo_release()
405 spin_lock(&svm_bo->list_lock); in svm_range_bo_release()
407 spin_unlock(&svm_bo->list_lock); in svm_range_bo_release()
409 if (mmget_not_zero(svm_bo->eviction_fence->mm)) { in svm_range_bo_release()
414 mm = svm_bo->eviction_fence->mm; in svm_range_bo_release()
421 pdd = kfd_get_process_device_data(svm_bo->node, p); in svm_range_bo_release()
423 atomic64_sub(amdgpu_bo_size(svm_bo->bo), &pdd->vram_usage); in svm_range_bo_release()
429 if (!dma_fence_is_signaled(&svm_bo->eviction_fence->base)) in svm_range_bo_release()
431 dma_fence_signal(&svm_bo->eviction_fence->base); in svm_range_bo_release()
432 dma_fence_put(&svm_bo->eviction_fence->base); in svm_range_bo_release()
433 amdgpu_bo_unref(&svm_bo->bo); in svm_range_bo_release()
442 svm_range_bo_release(&svm_bo->kref); in svm_range_bo_wq_release()
451 INIT_WORK(&svm_bo->release_work, svm_range_bo_wq_release); in svm_range_bo_release_async()
452 schedule_work(&svm_bo->release_work); in svm_range_bo_release_async()
457 kref_put(&svm_bo->kref, svm_range_bo_release_async); in svm_range_bo_unref_async()
463 kref_put(&svm_bo->kref, svm_range_bo_release); in svm_range_bo_unref()
469 mutex_lock(&prange->lock); in svm_range_validate_svm_bo()
470 if (!prange->svm_bo) { in svm_range_validate_svm_bo()
471 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
474 if (prange->ttm_res) { in svm_range_validate_svm_bo()
476 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
479 if (svm_bo_ref_unless_zero(prange->svm_bo)) { in svm_range_validate_svm_bo()
481 * Migrate from GPU to GPU, remove range from source svm_bo->node in svm_range_validate_svm_bo()
485 if (prange->svm_bo->node != node) { in svm_range_validate_svm_bo()
486 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
488 spin_lock(&prange->svm_bo->list_lock); in svm_range_validate_svm_bo()
489 list_del_init(&prange->svm_bo_list); in svm_range_validate_svm_bo()
490 spin_unlock(&prange->svm_bo->list_lock); in svm_range_validate_svm_bo()
492 svm_range_bo_unref(prange->svm_bo); in svm_range_validate_svm_bo()
495 if (READ_ONCE(prange->svm_bo->evicting)) { in svm_range_validate_svm_bo()
501 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
502 svm_bo = prange->svm_bo; in svm_range_validate_svm_bo()
503 f = dma_fence_get(&svm_bo->eviction_fence->base); in svm_range_validate_svm_bo()
504 svm_range_bo_unref(prange->svm_bo); in svm_range_validate_svm_bo()
505 /* wait for the fence to avoid long spin-loop in svm_range_validate_svm_bo()
514 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
516 prange->svms, prange->start, prange->last); in svm_range_validate_svm_bo()
518 prange->ttm_res = prange->svm_bo->bo->tbo.resource; in svm_range_validate_svm_bo()
523 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
526 /* We need a new svm_bo. Spin-loop to wait for concurrent in svm_range_validate_svm_bo()
528 * its range list and set prange->svm_bo to null. After this, in svm_range_validate_svm_bo()
531 while (!list_empty_careful(&prange->svm_bo_list) || prange->svm_bo) in svm_range_validate_svm_bo()
545 kref_init(&svm_bo->kref); in svm_range_bo_new()
546 INIT_LIST_HEAD(&svm_bo->range_list); in svm_range_bo_new()
547 spin_lock_init(&svm_bo->list_lock); in svm_range_bo_new()
565 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_vram_node_new()
566 pr_debug("pasid: %x svms 0x%p [0x%lx 0x%lx]\n", p->pasid, prange->svms, in svm_range_vram_node_new()
567 prange->start, prange->last); in svm_range_vram_node_new()
575 return -ENOMEM; in svm_range_vram_node_new()
577 mm = get_task_mm(p->lead_thread); in svm_range_vram_node_new()
581 return -ESRCH; in svm_range_vram_node_new()
583 svm_bo->node = node; in svm_range_vram_node_new()
584 svm_bo->eviction_fence = in svm_range_vram_node_new()
589 INIT_WORK(&svm_bo->eviction_work, svm_range_evict_svm_bo_worker); in svm_range_vram_node_new()
590 svm_bo->evicting = 0; in svm_range_vram_node_new()
592 bp.size = prange->npages * PAGE_SIZE; in svm_range_vram_node_new()
600 if (node->xcp) in svm_range_vram_node_new()
601 bp.xcp_id_plus1 = node->xcp->id + 1; in svm_range_vram_node_new()
603 r = amdgpu_bo_create_user(node->adev, &bp, &ubo); in svm_range_vram_node_new()
608 bo = &ubo->bo; in svm_range_vram_node_new()
611 bo->tbo.resource->start << PAGE_SHIFT, bp.size, in svm_range_vram_node_new()
612 bp.xcp_id_plus1 - 1); in svm_range_vram_node_new()
629 r = dma_resv_reserve_fences(bo->tbo.base.resv, 1); in svm_range_vram_node_new()
635 amdgpu_bo_fence(bo, &svm_bo->eviction_fence->base, true); in svm_range_vram_node_new()
639 svm_bo->bo = bo; in svm_range_vram_node_new()
640 prange->svm_bo = svm_bo; in svm_range_vram_node_new()
641 prange->ttm_res = bo->tbo.resource; in svm_range_vram_node_new()
642 prange->offset = 0; in svm_range_vram_node_new()
644 spin_lock(&svm_bo->list_lock); in svm_range_vram_node_new()
645 list_add(&prange->svm_bo_list, &svm_bo->range_list); in svm_range_vram_node_new()
646 spin_unlock(&svm_bo->list_lock); in svm_range_vram_node_new()
650 atomic64_add(amdgpu_bo_size(bo), &pdd->vram_usage); in svm_range_vram_node_new()
657 dma_fence_put(&svm_bo->eviction_fence->base); in svm_range_vram_node_new()
659 prange->ttm_res = NULL; in svm_range_vram_node_new()
666 /* serialize prange->svm_bo unref */ in svm_range_vram_node_free()
667 mutex_lock(&prange->lock); in svm_range_vram_node_free()
668 /* prange->svm_bo has not been unref */ in svm_range_vram_node_free()
669 if (prange->ttm_res) { in svm_range_vram_node_free()
670 prange->ttm_res = NULL; in svm_range_vram_node_free()
671 mutex_unlock(&prange->lock); in svm_range_vram_node_free()
672 svm_range_bo_unref(prange->svm_bo); in svm_range_vram_node_free()
674 mutex_unlock(&prange->lock); in svm_range_vram_node_free()
683 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_get_node_by_id()
690 return pdd->dev; in svm_range_get_node_by_id()
698 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_get_pdd_by_node()
709 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in svm_range_bo_validate()
745 return -EINVAL; in svm_range_check_attr()
750 return -EINVAL; in svm_range_check_attr()
752 !test_bit(gpuidx, p->svms.bitmap_supported)) { in svm_range_check_attr()
754 return -EINVAL; in svm_range_check_attr()
772 prange->preferred_loc = attrs[i].value; in svm_range_apply_attrs()
775 prange->prefetch_loc = attrs[i].value; in svm_range_apply_attrs()
780 if (!p->xnack_enabled) in svm_range_apply_attrs()
786 bitmap_clear(prange->bitmap_access, gpuidx, 1); in svm_range_apply_attrs()
787 bitmap_clear(prange->bitmap_aip, gpuidx, 1); in svm_range_apply_attrs()
789 bitmap_set(prange->bitmap_access, gpuidx, 1); in svm_range_apply_attrs()
790 bitmap_clear(prange->bitmap_aip, gpuidx, 1); in svm_range_apply_attrs()
792 bitmap_clear(prange->bitmap_access, gpuidx, 1); in svm_range_apply_attrs()
793 bitmap_set(prange->bitmap_aip, gpuidx, 1); in svm_range_apply_attrs()
798 prange->flags |= attrs[i].value; in svm_range_apply_attrs()
802 prange->flags &= ~attrs[i].value; in svm_range_apply_attrs()
805 prange->granularity = min_t(uint32_t, attrs[i].value, 0x3F); in svm_range_apply_attrs()
823 if (prange->preferred_loc != attrs[i].value) in svm_range_is_same_attrs()
837 if (test_bit(gpuidx, prange->bitmap_access) || in svm_range_is_same_attrs()
838 test_bit(gpuidx, prange->bitmap_aip)) in svm_range_is_same_attrs()
841 if (!test_bit(gpuidx, prange->bitmap_access)) in svm_range_is_same_attrs()
844 if (!test_bit(gpuidx, prange->bitmap_aip)) in svm_range_is_same_attrs()
849 if ((prange->flags & attrs[i].value) != attrs[i].value) in svm_range_is_same_attrs()
853 if ((prange->flags & attrs[i].value) != 0) in svm_range_is_same_attrs()
857 if (prange->granularity != attrs[i].value) in svm_range_is_same_attrs()
869 * svm_range_debug_dump - print all range information from svms
875 * Context: The caller must hold svms->lock
885 list_for_each_entry(prange, &svms->list, list) { in svm_range_debug_dump()
887 prange, prange->start, prange->npages, in svm_range_debug_dump()
888 prange->start + prange->npages - 1, in svm_range_debug_dump()
889 prange->actual_loc); in svm_range_debug_dump()
894 node = interval_tree_iter_first(&svms->objects, 0, ~0ULL); in svm_range_debug_dump()
898 prange, prange->start, prange->npages, in svm_range_debug_dump()
899 prange->start + prange->npages - 1, in svm_range_debug_dump()
900 prange->actual_loc); in svm_range_debug_dump()
940 if (!src->dma_addr[i]) in svm_range_copy_dma_addrs()
942 dst->dma_addr[i] = svm_range_copy_array(src->dma_addr[i], in svm_range_copy_dma_addrs()
943 sizeof(*src->dma_addr[i]), src->npages, 0, NULL); in svm_range_copy_dma_addrs()
944 if (!dst->dma_addr[i]) in svm_range_copy_dma_addrs()
945 return -ENOMEM; in svm_range_copy_dma_addrs()
965 d = (new_start - old_start) * size; in svm_range_split_array()
969 return -ENOMEM; in svm_range_split_array()
974 return -ENOMEM; in svm_range_split_array()
987 uint64_t npages = last - start + 1; in svm_range_split_pages()
991 r = svm_range_split_array(&new->dma_addr[i], &old->dma_addr[i], in svm_range_split_pages()
992 sizeof(*old->dma_addr[i]), old->start, in svm_range_split_pages()
993 npages, new->start, new->npages, in svm_range_split_pages()
994 old->actual_loc ? &new->vram_pages : NULL); in svm_range_split_pages()
998 if (old->actual_loc) in svm_range_split_pages()
999 old->vram_pages -= new->vram_pages; in svm_range_split_pages()
1008 uint64_t npages = last - start + 1; in svm_range_split_nodes()
1011 new->svms, new, new->start, start, last); in svm_range_split_nodes()
1013 if (new->start == old->start) { in svm_range_split_nodes()
1014 new->offset = old->offset; in svm_range_split_nodes()
1015 old->offset += new->npages; in svm_range_split_nodes()
1017 new->offset = old->offset + npages; in svm_range_split_nodes()
1020 new->svm_bo = svm_range_bo_ref(old->svm_bo); in svm_range_split_nodes()
1021 new->ttm_res = old->ttm_res; in svm_range_split_nodes()
1023 spin_lock(&new->svm_bo->list_lock); in svm_range_split_nodes()
1024 list_add(&new->svm_bo_list, &new->svm_bo->range_list); in svm_range_split_nodes()
1025 spin_unlock(&new->svm_bo->list_lock); in svm_range_split_nodes()
1031 * svm_range_split_adjust - split range and adjust
1039 * range from new_start up to size new->npages, the remaining old range is from
1043 * 0 - OK, -ENOMEM - out of memory
1052 new->svms, new->start, old->start, old->last, start, last); in svm_range_split_adjust()
1054 if (new->start < old->start || in svm_range_split_adjust()
1055 new->last > old->last) { in svm_range_split_adjust()
1057 return -EINVAL; in svm_range_split_adjust()
1064 if (old->actual_loc && old->ttm_res) { in svm_range_split_adjust()
1070 old->npages = last - start + 1; in svm_range_split_adjust()
1071 old->start = start; in svm_range_split_adjust()
1072 old->last = last; in svm_range_split_adjust()
1073 new->flags = old->flags; in svm_range_split_adjust()
1074 new->preferred_loc = old->preferred_loc; in svm_range_split_adjust()
1075 new->prefetch_loc = old->prefetch_loc; in svm_range_split_adjust()
1076 new->actual_loc = old->actual_loc; in svm_range_split_adjust()
1077 new->granularity = old->granularity; in svm_range_split_adjust()
1078 new->mapped_to_gpu = old->mapped_to_gpu; in svm_range_split_adjust()
1079 bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE); in svm_range_split_adjust()
1080 bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_split_adjust()
1081 atomic_set(&new->queue_refcount, atomic_read(&old->queue_refcount)); in svm_range_split_adjust()
1087 * svm_range_split - split a range in 2 ranges
1095 * case 1: if start == prange->start
1097 * new range [last + 1, prange->last]
1099 * case 2: if last == prange->last
1101 * new range [prange->start, start - 1]
1104 * 0 - OK, -ENOMEM - out of memory, -EINVAL - invalid start, last
1110 uint64_t old_start = prange->start; in svm_range_split()
1111 uint64_t old_last = prange->last; in svm_range_split()
1115 pr_debug("svms 0x%p [0x%llx 0x%llx] to [0x%llx 0x%llx]\n", prange->svms, in svm_range_split()
1119 return -EINVAL; in svm_range_split()
1121 return -EINVAL; in svm_range_split()
1123 svms = prange->svms; in svm_range_split()
1127 *new = svm_range_new(svms, old_start, start - 1, false); in svm_range_split()
1129 return -ENOMEM; in svm_range_split()
1147 int r = svm_range_split(prange, prange->start, new_last, &tail); in svm_range_split_tail()
1150 list_add(&tail->list, insert_list); in svm_range_split_tail()
1151 if (!IS_ALIGNED(new_last + 1, 1UL << prange->granularity)) in svm_range_split_tail()
1152 list_add(&tail->update_list, remap_list); in svm_range_split_tail()
1162 int r = svm_range_split(prange, new_start, prange->last, &head); in svm_range_split_head()
1165 list_add(&head->list, insert_list); in svm_range_split_head()
1166 if (!IS_ALIGNED(new_start, 1UL << prange->granularity)) in svm_range_split_head()
1167 list_add(&head->update_list, remap_list); in svm_range_split_head()
1177 pchild, pchild->start, pchild->last, prange, op); in svm_range_add_child()
1179 pchild->work_item.mm = mm; in svm_range_add_child()
1180 pchild->work_item.op = op; in svm_range_add_child()
1181 list_add_tail(&pchild->child_list, &prange->child_list); in svm_range_add_child()
1187 return (node_a->adev == node_b->adev || in svm_nodes_in_same_hive()
1188 amdgpu_xgmi_same_hive(node_a->adev, node_b->adev)); in svm_nodes_in_same_hive()
1196 uint32_t flags = prange->flags; in svm_range_get_pte_flags()
1206 bo_node = prange->svm_bo->node; in svm_range_get_pte_flags()
1230 if (node->adev->gmc.xgmi.connected_to_cpu) in svm_range_get_pte_flags()
1247 mtype_local = (gc_ip_version < IP_VERSION(9, 5, 0) && !node->adev->rev_id) ? in svm_range_get_pte_flags()
1255 if (bo_node->adev == node->adev && in svm_range_get_pte_flags()
1256 (!bo_node->xcp || !node->xcp || bo_node->xcp->mem_id == node->xcp->mem_id)) in svm_range_get_pte_flags()
1263 /* PCIe P2P on GPUs pre-9.5.0 */ in svm_range_get_pte_flags()
1271 } else if (node->adev->flags & AMD_IS_APU) { in svm_range_get_pte_flags()
1272 /* On NUMA systems, locality is determined per-page in svm_range_get_pte_flags()
1309 pte_flags |= amdgpu_gem_va_map_flags(node->adev, mapping_flags); in svm_range_get_pte_flags()
1338 if (!prange->mapped_to_gpu) { in svm_range_unmap_from_gpus()
1340 prange, prange->start, prange->last); in svm_range_unmap_from_gpus()
1344 if (prange->start == start && prange->last == last) { in svm_range_unmap_from_gpus()
1345 pr_debug("unmap svms 0x%p prange 0x%p\n", prange->svms, prange); in svm_range_unmap_from_gpus()
1346 prange->mapped_to_gpu = false; in svm_range_unmap_from_gpus()
1349 bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip, in svm_range_unmap_from_gpus()
1351 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_unmap_from_gpus()
1358 return -EINVAL; in svm_range_unmap_from_gpus()
1361 kfd_smi_event_unmap_from_gpu(pdd->dev, p->lead_thread->pid, in svm_range_unmap_from_gpus()
1364 r = svm_range_unmap_from_gpu(pdd->dev->adev, in svm_range_unmap_from_gpus()
1365 drm_priv_to_vm(pdd->drm_priv), in svm_range_unmap_from_gpus()
1389 struct amdgpu_device *adev = pdd->dev->adev; in svm_range_map_to_gpu()
1390 struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv); in svm_range_map_to_gpu()
1397 last_start = prange->start + offset; in svm_range_map_to_gpu()
1399 pr_debug("svms 0x%p [0x%lx 0x%lx] readonly %d\n", prange->svms, in svm_range_map_to_gpu()
1400 last_start, last_start + npages - 1, readonly); in svm_range_map_to_gpu()
1409 if (i < offset + npages - 1 && in svm_range_map_to_gpu()
1414 last_start, prange->start + i, last_domain ? "GPU" : "CPU"); in svm_range_map_to_gpu()
1416 pte_flags = svm_range_get_pte_flags(pdd->dev, prange, last_domain); in svm_range_map_to_gpu()
1421 prange->svms, last_start, prange->start + i, in svm_range_map_to_gpu()
1430 NULL, last_start, prange->start + i, in svm_range_map_to_gpu()
1432 (last_start - prange->start) << PAGE_SHIFT, in svm_range_map_to_gpu()
1433 bo_adev ? bo_adev->vm_manager.vram_base_offset : 0, in svm_range_map_to_gpu()
1434 NULL, dma_addr, &vm->last_update); in svm_range_map_to_gpu()
1436 for (j = last_start - prange->start; j <= i; j++) in svm_range_map_to_gpu()
1440 pr_debug("failed %d to map to gpu 0x%lx\n", r, prange->start); in svm_range_map_to_gpu()
1443 last_start = prange->start + i + 1; in svm_range_map_to_gpu()
1449 prange->start); in svm_range_map_to_gpu()
1454 *fence = dma_fence_get(vm->last_update); in svm_range_map_to_gpu()
1472 if (prange->svm_bo && prange->ttm_res) in svm_range_map_to_gpus()
1473 bo_adev = prange->svm_bo->node->adev; in svm_range_map_to_gpus()
1475 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_map_to_gpus()
1481 return -EINVAL; in svm_range_map_to_gpus()
1484 pdd = kfd_bind_process_to_device(pdd->dev, p); in svm_range_map_to_gpus()
1486 return -EINVAL; in svm_range_map_to_gpus()
1488 if (bo_adev && pdd->dev->adev != bo_adev && in svm_range_map_to_gpus()
1489 !amdgpu_xgmi_same_hive(pdd->dev->adev, bo_adev)) { in svm_range_map_to_gpus()
1495 prange->dma_addr[gpuidx], in svm_range_map_to_gpus()
1532 drm_exec_init(&ctx->exec, intr ? DRM_EXEC_INTERRUPTIBLE_WAIT: 0, 0); in svm_range_reserve_bos()
1533 drm_exec_until_all_locked(&ctx->exec) { in svm_range_reserve_bos()
1534 for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) { in svm_range_reserve_bos()
1535 pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx); in svm_range_reserve_bos()
1538 r = -EINVAL; in svm_range_reserve_bos()
1541 vm = drm_priv_to_vm(pdd->drm_priv); in svm_range_reserve_bos()
1543 r = amdgpu_vm_lock_pd(vm, &ctx->exec, 2); in svm_range_reserve_bos()
1544 drm_exec_retry_on_contention(&ctx->exec); in svm_range_reserve_bos()
1552 for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) { in svm_range_reserve_bos()
1553 pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx); in svm_range_reserve_bos()
1556 r = -EINVAL; in svm_range_reserve_bos()
1560 r = amdgpu_vm_validate(pdd->dev->adev, in svm_range_reserve_bos()
1561 drm_priv_to_vm(pdd->drm_priv), NULL, in svm_range_reserve_bos()
1572 drm_exec_fini(&ctx->exec); in svm_range_reserve_bos()
1578 drm_exec_fini(&ctx->exec); in svm_range_unreserve_bos()
1589 return SVM_ADEV_PGMAP_OWNER(pdd->dev->adev); in kfd_svm_page_owner()
1599 * prange->migrate_mutex must be held.
1604 * The following sequence ensures race-free validation and GPU mapping:
1608 * 3. DMA-map pages (if system memory)
1609 * 4-a. Take notifier lock
1610 * 4-b. Check that pages still valid (mmu_interval_read_retry)
1611 * 4-c. Check that the range was not split or otherwise invalidated
1612 * 4-d. Update GPU page table
1630 return -ENOMEM; in svm_range_validate_and_map()
1631 ctx->process = container_of(prange->svms, struct kfd_process, svms); in svm_range_validate_and_map()
1632 ctx->prange = prange; in svm_range_validate_and_map()
1633 ctx->intr = intr; in svm_range_validate_and_map()
1636 bitmap_zero(ctx->bitmap, MAX_GPU_INSTANCE); in svm_range_validate_and_map()
1637 bitmap_set(ctx->bitmap, gpuidx, 1); in svm_range_validate_and_map()
1638 } else if (ctx->process->xnack_enabled) { in svm_range_validate_and_map()
1639 bitmap_copy(ctx->bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_validate_and_map()
1642 * GPU, which has ACCESS attribute to the range, create mapping in svm_range_validate_and_map()
1645 if (prange->actual_loc) { in svm_range_validate_and_map()
1646 gpuidx = kfd_process_gpuidx_from_gpuid(ctx->process, in svm_range_validate_and_map()
1647 prange->actual_loc); in svm_range_validate_and_map()
1650 prange->actual_loc); in svm_range_validate_and_map()
1651 r = -EINVAL; in svm_range_validate_and_map()
1654 if (test_bit(gpuidx, prange->bitmap_access)) in svm_range_validate_and_map()
1655 bitmap_set(ctx->bitmap, gpuidx, 1); in svm_range_validate_and_map()
1660 * update mapping on GPUs with ACCESS attribute in svm_range_validate_and_map()
1662 if (bitmap_empty(ctx->bitmap, MAX_GPU_INSTANCE)) { in svm_range_validate_and_map()
1663 if (prange->mapped_to_gpu || in svm_range_validate_and_map()
1664 prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED) in svm_range_validate_and_map()
1665 bitmap_copy(ctx->bitmap, prange->bitmap_access, MAX_GPU_INSTANCE); in svm_range_validate_and_map()
1668 bitmap_or(ctx->bitmap, prange->bitmap_access, in svm_range_validate_and_map()
1669 prange->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_validate_and_map()
1672 if (bitmap_empty(ctx->bitmap, MAX_GPU_INSTANCE)) { in svm_range_validate_and_map()
1677 if (prange->actual_loc && !prange->ttm_res) { in svm_range_validate_and_map()
1682 r = -EINVAL; in svm_range_validate_and_map()
1690 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_validate_and_map()
1691 owner = kfd_svm_page_owner(p, find_first_bit(ctx->bitmap, in svm_range_validate_and_map()
1693 for_each_set_bit(idx, ctx->bitmap, MAX_GPU_INSTANCE) { in svm_range_validate_and_map()
1714 readonly = !(vma->vm_flags & VM_WRITE); in svm_range_validate_and_map()
1716 next = min(vma->vm_end, end); in svm_range_validate_and_map()
1717 npages = (next - addr) >> PAGE_SHIFT; in svm_range_validate_and_map()
1718 WRITE_ONCE(p->svms.faulting_task, current); in svm_range_validate_and_map()
1719 r = amdgpu_hmm_range_get_pages(&prange->notifier, addr, npages, in svm_range_validate_and_map()
1722 WRITE_ONCE(p->svms.faulting_task, NULL); in svm_range_validate_and_map()
1726 r = -EFAULT; in svm_range_validate_and_map()
1730 offset = (addr >> PAGE_SHIFT) - prange->start; in svm_range_validate_and_map()
1731 r = svm_range_dma_map(prange, ctx->bitmap, offset, npages, in svm_range_validate_and_map()
1732 hmm_range->hmm_pfns); in svm_range_validate_and_map()
1745 r = -EAGAIN; in svm_range_validate_and_map()
1748 if (!r && !list_empty(&prange->child_list)) { in svm_range_validate_and_map()
1750 r = -EAGAIN; in svm_range_validate_and_map()
1754 map_start_vma = max(map_start, prange->start + offset); in svm_range_validate_and_map()
1755 map_last_vma = min(map_last, prange->start + offset + npages - 1); in svm_range_validate_and_map()
1757 offset = map_start_vma - prange->start; in svm_range_validate_and_map()
1758 npages = map_last_vma - map_start_vma + 1; in svm_range_validate_and_map()
1760 ctx->bitmap, wait, flush_tlb); in svm_range_validate_and_map()
1765 prange->mapped_to_gpu = true; in svm_range_validate_and_map()
1774 prange->validate_timestamp = ktime_get_boottime(); in svm_range_validate_and_map()
1783 * svm_range_list_lock_and_flush_work - flush pending deferred work
1796 flush_work(&svms->deferred_list_work); in svm_range_list_lock_and_flush_work()
1799 if (list_empty(&svms->deferred_range_list)) in svm_range_list_lock_and_flush_work()
1819 evicted_ranges = atomic_read(&svms->evicted_ranges); in svm_range_restore_work()
1826 process_info = p->kgd_process_info; in svm_range_restore_work()
1829 mm = get_task_mm(p->lead_thread); in svm_range_restore_work()
1835 mutex_lock(&process_info->lock); in svm_range_restore_work()
1837 mutex_lock(&svms->lock); in svm_range_restore_work()
1839 evicted_ranges = atomic_read(&svms->evicted_ranges); in svm_range_restore_work()
1841 list_for_each_entry(prange, &svms->list, list) { in svm_range_restore_work()
1842 invalid = atomic_read(&prange->invalid); in svm_range_restore_work()
1847 prange->svms, prange, prange->start, prange->last, in svm_range_restore_work()
1853 mutex_lock(&prange->migrate_mutex); in svm_range_restore_work()
1855 r = svm_range_validate_and_map(mm, prange->start, prange->last, prange, in svm_range_restore_work()
1859 prange->start); in svm_range_restore_work()
1861 mutex_unlock(&prange->migrate_mutex); in svm_range_restore_work()
1865 if (atomic_cmpxchg(&prange->invalid, invalid, 0) != invalid) in svm_range_restore_work()
1869 if (atomic_cmpxchg(&svms->evicted_ranges, evicted_ranges, 0) != in svm_range_restore_work()
1886 mutex_unlock(&svms->lock); in svm_range_restore_work()
1888 mutex_unlock(&process_info->lock); in svm_range_restore_work()
1893 queue_delayed_work(system_freezable_wq, &svms->restore_work, in svm_range_restore_work()
1902 * svm_range_evict - evict svm range
1909 * Stop all queues of the process to ensure GPU doesn't access the memory, then
1922 struct svm_range_list *svms = prange->svms; in svm_range_evict()
1930 svms, prange->start, prange->last, start, last); in svm_range_evict()
1932 if (!p->xnack_enabled || in svm_range_evict()
1933 (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) { in svm_range_evict()
1935 bool mapped = prange->mapped_to_gpu; in svm_range_evict()
1937 list_for_each_entry(pchild, &prange->child_list, child_list) { in svm_range_evict()
1938 if (!pchild->mapped_to_gpu) in svm_range_evict()
1941 mutex_lock_nested(&pchild->lock, 1); in svm_range_evict()
1942 if (pchild->start <= last && pchild->last >= start) { in svm_range_evict()
1944 pchild->start, pchild->last); in svm_range_evict()
1945 atomic_inc(&pchild->invalid); in svm_range_evict()
1947 mutex_unlock(&pchild->lock); in svm_range_evict()
1953 if (prange->start <= last && prange->last >= start) in svm_range_evict()
1954 atomic_inc(&prange->invalid); in svm_range_evict()
1956 evicted_ranges = atomic_inc_return(&svms->evicted_ranges); in svm_range_evict()
1961 prange->svms, prange->start, prange->last); in svm_range_evict()
1969 queue_delayed_work(system_freezable_wq, &svms->restore_work, in svm_range_evict()
1981 prange->svms, start, last); in svm_range_evict()
1982 list_for_each_entry(pchild, &prange->child_list, child_list) { in svm_range_evict()
1983 mutex_lock_nested(&pchild->lock, 1); in svm_range_evict()
1984 s = max(start, pchild->start); in svm_range_evict()
1985 l = min(last, pchild->last); in svm_range_evict()
1988 mutex_unlock(&pchild->lock); in svm_range_evict()
1990 s = max(start, prange->start); in svm_range_evict()
1991 l = min(last, prange->last); in svm_range_evict()
2003 new = svm_range_new(old->svms, old->start, old->last, false); in svm_range_clone()
2010 if (old->svm_bo) { in svm_range_clone()
2011 new->ttm_res = old->ttm_res; in svm_range_clone()
2012 new->offset = old->offset; in svm_range_clone()
2013 new->svm_bo = svm_range_bo_ref(old->svm_bo); in svm_range_clone()
2014 spin_lock(&new->svm_bo->list_lock); in svm_range_clone()
2015 list_add(&new->svm_bo_list, &new->svm_bo->range_list); in svm_range_clone()
2016 spin_unlock(&new->svm_bo->list_lock); in svm_range_clone()
2018 new->flags = old->flags; in svm_range_clone()
2019 new->preferred_loc = old->preferred_loc; in svm_range_clone()
2020 new->prefetch_loc = old->prefetch_loc; in svm_range_clone()
2021 new->actual_loc = old->actual_loc; in svm_range_clone()
2022 new->granularity = old->granularity; in svm_range_clone()
2023 new->mapped_to_gpu = old->mapped_to_gpu; in svm_range_clone()
2024 new->vram_pages = old->vram_pages; in svm_range_clone()
2025 bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE); in svm_range_clone()
2026 bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_clone()
2027 atomic_set(&new->queue_refcount, atomic_read(&old->queue_refcount)); in svm_range_clone()
2039 for (i = 0; i < adev->kfd.dev->num_nodes; i++) { in svm_range_set_max_pages()
2040 if (adev->kfd.dev->nodes[i]->xcp) in svm_range_set_max_pages()
2041 id = adev->kfd.dev->nodes[i]->xcp->id; in svm_range_set_max_pages()
2043 id = -1; in svm_range_set_max_pages()
2068 l = min(last, ALIGN_DOWN(start + max_pages, max_pages) - 1); in svm_range_split_new()
2072 return -ENOMEM; in svm_range_split_new()
2073 list_add(&prange->list, insert_list); in svm_range_split_new()
2074 list_add(&prange->update_list, update_list); in svm_range_split_new()
2082 * svm_range_add - add svm range and handle overlap
2106 * Context: Process context, caller must hold svms->lock
2109 * 0 - OK, otherwise error code
2117 unsigned long last = start + size - 1UL; in svm_range_add()
2118 struct svm_range_list *svms = &p->svms; in svm_range_add()
2125 pr_debug("svms 0x%p [0x%llx 0x%lx]\n", &p->svms, start, last); in svm_range_add()
2133 node = interval_tree_iter_first(&svms->objects, start, last); in svm_range_add()
2138 pr_debug("found overlap node [0x%lx 0x%lx]\n", node->start, in svm_range_add()
2139 node->last); in svm_range_add()
2143 next_start = min(node->last, last) + 1; in svm_range_add()
2146 prange->mapped_to_gpu) { in svm_range_add()
2148 } else if (node->start < start || node->last > last) { in svm_range_add()
2157 r = -ENOMEM; in svm_range_add()
2161 list_add(&old->update_list, remove_list); in svm_range_add()
2162 list_add(&prange->list, insert_list); in svm_range_add()
2163 list_add(&prange->update_list, update_list); in svm_range_add()
2165 if (node->start < start) { in svm_range_add()
2172 if (node->last > last) { in svm_range_add()
2183 list_add(&prange->update_list, update_list); in svm_range_add()
2187 if (node->start > start) { in svm_range_add()
2188 r = svm_range_split_new(svms, start, node->start - 1, in svm_range_add()
2225 start = prange->notifier.interval_tree.start >> PAGE_SHIFT; in svm_range_update_notifier_and_interval_tree()
2226 last = prange->notifier.interval_tree.last >> PAGE_SHIFT; in svm_range_update_notifier_and_interval_tree()
2228 if (prange->start == start && prange->last == last) in svm_range_update_notifier_and_interval_tree()
2232 prange->svms, prange, start, last, prange->start, in svm_range_update_notifier_and_interval_tree()
2233 prange->last); in svm_range_update_notifier_and_interval_tree()
2236 interval_tree_remove(&prange->it_node, &prange->svms->objects); in svm_range_update_notifier_and_interval_tree()
2239 prange->it_node.start = prange->start; in svm_range_update_notifier_and_interval_tree()
2240 prange->it_node.last = prange->last; in svm_range_update_notifier_and_interval_tree()
2242 interval_tree_insert(&prange->it_node, &prange->svms->objects); in svm_range_update_notifier_and_interval_tree()
2250 switch (prange->work_item.op) { in svm_range_handle_list_op()
2253 svms, prange, prange->start, prange->last); in svm_range_handle_list_op()
2257 svms, prange, prange->start, prange->last); in svm_range_handle_list_op()
2264 svms, prange, prange->start, prange->last); in svm_range_handle_list_op()
2269 svms, prange, prange->start, prange->last); in svm_range_handle_list_op()
2275 prange->start, prange->last); in svm_range_handle_list_op()
2281 prange, prange->start, prange->last); in svm_range_handle_list_op()
2288 prange->work_item.op); in svm_range_handle_list_op()
2300 for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) { in svm_range_drain_retry_fault()
2301 pdd = p->pdds[i]; in svm_range_drain_retry_fault()
2307 amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev, in svm_range_drain_retry_fault()
2308 pdd->dev->adev->irq.retry_cam_enabled ? in svm_range_drain_retry_fault()
2309 &pdd->dev->adev->irq.ih : in svm_range_drain_retry_fault()
2310 &pdd->dev->adev->irq.ih1); in svm_range_drain_retry_fault()
2312 if (pdd->dev->adev->irq.retry_cam_enabled) in svm_range_drain_retry_fault()
2313 amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev, in svm_range_drain_retry_fault()
2314 &pdd->dev->adev->irq.ih_soft); in svm_range_drain_retry_fault()
2330 spin_lock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2331 while (!list_empty(&svms->deferred_range_list)) { in svm_range_deferred_list_work()
2332 prange = list_first_entry(&svms->deferred_range_list, in svm_range_deferred_list_work()
2334 spin_unlock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2337 prange->start, prange->last, prange->work_item.op); in svm_range_deferred_list_work()
2339 mm = prange->work_item.mm; in svm_range_deferred_list_work()
2351 spin_lock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2352 list_del_init(&prange->deferred_list); in svm_range_deferred_list_work()
2353 spin_unlock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2355 mutex_lock(&svms->lock); in svm_range_deferred_list_work()
2356 mutex_lock(&prange->migrate_mutex); in svm_range_deferred_list_work()
2357 while (!list_empty(&prange->child_list)) { in svm_range_deferred_list_work()
2360 pchild = list_first_entry(&prange->child_list, in svm_range_deferred_list_work()
2363 pchild->work_item.op); in svm_range_deferred_list_work()
2364 list_del_init(&pchild->child_list); in svm_range_deferred_list_work()
2367 mutex_unlock(&prange->migrate_mutex); in svm_range_deferred_list_work()
2370 mutex_unlock(&svms->lock); in svm_range_deferred_list_work()
2378 spin_lock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2380 spin_unlock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2388 spin_lock(&svms->deferred_list_lock); in svm_range_add_list_work()
2390 if (!list_empty(&prange->deferred_list)) { in svm_range_add_list_work()
2392 WARN_ONCE(prange->work_item.mm != mm, "unmatch mm\n"); in svm_range_add_list_work()
2394 prange->work_item.op != SVM_OP_UNMAP_RANGE) in svm_range_add_list_work()
2395 prange->work_item.op = op; in svm_range_add_list_work()
2397 prange->work_item.op = op; in svm_range_add_list_work()
2401 prange->work_item.mm = mm; in svm_range_add_list_work()
2402 list_add_tail(&prange->deferred_list, in svm_range_add_list_work()
2403 &prange->svms->deferred_range_list); in svm_range_add_list_work()
2405 prange, prange->start, prange->last, op); in svm_range_add_list_work()
2407 spin_unlock(&svms->deferred_list_lock); in svm_range_add_list_work()
2412 spin_lock(&svms->deferred_list_lock); in schedule_deferred_list_work()
2413 if (!list_empty(&svms->deferred_range_list)) in schedule_deferred_list_work()
2414 schedule_work(&svms->deferred_list_work); in schedule_deferred_list_work()
2415 spin_unlock(&svms->deferred_list_lock); in schedule_deferred_list_work()
2426 if (prange->work_item.op == SVM_OP_UNMAP_RANGE) { in svm_range_unmap_split()
2428 prange->start, prange->last); in svm_range_unmap_split()
2431 if (start > prange->last || last < prange->start) in svm_range_unmap_split()
2435 if (start > prange->start) in svm_range_unmap_split()
2436 svm_range_split(prange, prange->start, start - 1, &tail); in svm_range_unmap_split()
2437 if (last < tail->last) in svm_range_unmap_split()
2438 svm_range_split(tail, last + 1, tail->last, &head); in svm_range_unmap_split()
2448 prange->work_item.op = SVM_OP_UNMAP_RANGE; in svm_range_unmap_split()
2464 if (atomic_read(&prange->queue_refcount)) { in svm_range_unmap_from_cpu()
2468 prange->start << PAGE_SHIFT); in svm_range_unmap_from_cpu()
2477 svms = &p->svms; in svm_range_unmap_from_cpu()
2480 prange, prange->start, prange->last, start, last); in svm_range_unmap_from_cpu()
2485 for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) { in svm_range_unmap_from_cpu()
2491 pdd = p->pdds[i]; in svm_range_unmap_from_cpu()
2495 adev = pdd->dev->adev; in svm_range_unmap_from_cpu()
2498 if (adev->irq.ih1.ring_size) { in svm_range_unmap_from_cpu()
2499 ih = &adev->irq.ih1; in svm_range_unmap_from_cpu()
2501 if (ih->rptr != checkpoint_wptr) { in svm_range_unmap_from_cpu()
2502 svms->checkpoint_ts[i] = in svm_range_unmap_from_cpu()
2503 amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1); in svm_range_unmap_from_cpu()
2508 /* check if dev->irq.ih_soft is not empty */ in svm_range_unmap_from_cpu()
2509 ih = &adev->irq.ih_soft; in svm_range_unmap_from_cpu()
2511 if (ih->rptr != checkpoint_wptr) in svm_range_unmap_from_cpu()
2512 svms->checkpoint_ts[i] = amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1); in svm_range_unmap_from_cpu()
2515 unmap_parent = start <= prange->start && last >= prange->last; in svm_range_unmap_from_cpu()
2517 list_for_each_entry(pchild, &prange->child_list, child_list) { in svm_range_unmap_from_cpu()
2518 mutex_lock_nested(&pchild->lock, 1); in svm_range_unmap_from_cpu()
2519 s = max(start, pchild->start); in svm_range_unmap_from_cpu()
2520 l = min(last, pchild->last); in svm_range_unmap_from_cpu()
2524 mutex_unlock(&pchild->lock); in svm_range_unmap_from_cpu()
2526 s = max(start, prange->start); in svm_range_unmap_from_cpu()
2527 l = min(last, prange->last); in svm_range_unmap_from_cpu()
2543 * svm_range_cpu_invalidate_pagetables - interval notifier callback
2571 if (range->event == MMU_NOTIFY_RELEASE) in svm_range_cpu_invalidate_pagetables()
2573 if (!mmget_not_zero(mni->mm)) in svm_range_cpu_invalidate_pagetables()
2576 start = mni->interval_tree.start; in svm_range_cpu_invalidate_pagetables()
2577 last = mni->interval_tree.last; in svm_range_cpu_invalidate_pagetables()
2578 start = max(start, range->start) >> PAGE_SHIFT; in svm_range_cpu_invalidate_pagetables()
2579 last = min(last, range->end - 1) >> PAGE_SHIFT; in svm_range_cpu_invalidate_pagetables()
2581 start, last, range->start >> PAGE_SHIFT, in svm_range_cpu_invalidate_pagetables()
2582 (range->end - 1) >> PAGE_SHIFT, in svm_range_cpu_invalidate_pagetables()
2583 mni->interval_tree.start >> PAGE_SHIFT, in svm_range_cpu_invalidate_pagetables()
2584 mni->interval_tree.last >> PAGE_SHIFT, range->event); in svm_range_cpu_invalidate_pagetables()
2591 switch (range->event) { in svm_range_cpu_invalidate_pagetables()
2593 svm_range_unmap_from_cpu(mni->mm, prange, start, last); in svm_range_cpu_invalidate_pagetables()
2596 svm_range_evict(prange, mni->mm, start, last, range->event); in svm_range_cpu_invalidate_pagetables()
2601 mmput(mni->mm); in svm_range_cpu_invalidate_pagetables()
2607 * svm_range_from_addr - find svm range from fault address
2612 * Context: The caller must hold svms->lock
2624 node = interval_tree_iter_first(&svms->objects, addr, addr); in svm_range_from_addr()
2630 addr, prange->start, prange->last, node->start, node->last); in svm_range_from_addr()
2632 if (addr >= prange->start && addr <= prange->last) { in svm_range_from_addr()
2637 list_for_each_entry(pchild, &prange->child_list, child_list) in svm_range_from_addr()
2638 if (addr >= pchild->start && addr <= pchild->last) { in svm_range_from_addr()
2640 addr, pchild->start, pchild->last); in svm_range_from_addr()
2649 /* svm_range_best_restore_location - decide the best fault restore location
2664 * Otherwise, GPU no access, best_loc is -1.
2667 * -1 means vm fault GPU no access
2680 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_best_restore_location()
2685 return -1; in svm_range_best_restore_location()
2688 if (node->adev->flags & AMD_IS_APU) in svm_range_best_restore_location()
2691 if (prange->preferred_loc == gpuid || in svm_range_best_restore_location()
2692 prange->preferred_loc == KFD_IOCTL_SVM_LOCATION_SYSMEM) { in svm_range_best_restore_location()
2693 return prange->preferred_loc; in svm_range_best_restore_location()
2694 } else if (prange->preferred_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED) { in svm_range_best_restore_location()
2695 preferred_node = svm_range_get_node_by_id(prange, prange->preferred_loc); in svm_range_best_restore_location()
2697 return prange->preferred_loc; in svm_range_best_restore_location()
2701 if (test_bit(*gpuidx, prange->bitmap_access)) in svm_range_best_restore_location()
2704 if (test_bit(*gpuidx, prange->bitmap_aip)) { in svm_range_best_restore_location()
2705 if (!prange->actual_loc) in svm_range_best_restore_location()
2708 bo_node = svm_range_get_node_by_id(prange, prange->actual_loc); in svm_range_best_restore_location()
2710 return prange->actual_loc; in svm_range_best_restore_location()
2715 return -1; in svm_range_best_restore_location()
2728 vma = vma_lookup(p->mm, addr << PAGE_SHIFT); in svm_range_get_range_boundaries()
2731 return -EFAULT; in svm_range_get_range_boundaries()
2736 start_limit = max(vma->vm_start >> PAGE_SHIFT, in svm_range_get_range_boundaries()
2737 (unsigned long)ALIGN_DOWN(addr, 1UL << p->svms.default_granularity)); in svm_range_get_range_boundaries()
2738 end_limit = min(vma->vm_end >> PAGE_SHIFT, in svm_range_get_range_boundaries()
2739 (unsigned long)ALIGN(addr + 1, 1UL << p->svms.default_granularity)); in svm_range_get_range_boundaries()
2742 node = interval_tree_iter_first(&p->svms.objects, addr + 1, ULONG_MAX); in svm_range_get_range_boundaries()
2744 end_limit = min(end_limit, node->start); in svm_range_get_range_boundaries()
2746 rb_node = rb_prev(&node->rb); in svm_range_get_range_boundaries()
2751 rb_node = rb_last(&p->svms.objects.rb_root); in svm_range_get_range_boundaries()
2755 if (node->last >= addr) { in svm_range_get_range_boundaries()
2757 return -EFAULT; in svm_range_get_range_boundaries()
2759 start_limit = max(start_limit, node->last + 1); in svm_range_get_range_boundaries()
2763 *last = end_limit - 1; in svm_range_get_range_boundaries()
2766 vma->vm_start >> PAGE_SHIFT, vma->vm_end >> PAGE_SHIFT, in svm_range_get_range_boundaries()
2783 for (i = 0; i < p->n_pdds; i++) { in svm_range_check_vm_userptr()
2786 if (!p->pdds[i]->drm_priv) in svm_range_check_vm_userptr()
2789 vm = drm_priv_to_vm(p->pdds[i]->drm_priv); in svm_range_check_vm_userptr()
2790 r = amdgpu_bo_reserve(vm->root.bo, false); in svm_range_check_vm_userptr()
2794 /* Check userptr by searching entire vm->va interval tree */ in svm_range_check_vm_userptr()
2795 node = interval_tree_iter_first(&vm->va, 0, ~0ULL); in svm_range_check_vm_userptr()
2799 bo = mapping->bo_va->base.bo; in svm_range_check_vm_userptr()
2801 if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, in svm_range_check_vm_userptr()
2813 *bo_l = *bo_s + bo->tbo.ttm->num_pages - 1; in svm_range_check_vm_userptr()
2815 amdgpu_bo_unreserve(vm->root.bo); in svm_range_check_vm_userptr()
2816 return -EADDRINUSE; in svm_range_check_vm_userptr()
2818 amdgpu_bo_unreserve(vm->root.bo); in svm_range_check_vm_userptr()
2842 if (r != -EADDRINUSE) in svm_range_create_unregistered_range()
2845 if (r == -EADDRINUSE) { in svm_range_create_unregistered_range()
2854 prange = svm_range_new(&p->svms, start, last, true); in svm_range_create_unregistered_range()
2866 prange->preferred_loc = KFD_IOCTL_SVM_LOCATION_SYSMEM; in svm_range_create_unregistered_range()
2874 /* svm_range_skip_recover - decide if prange can be recovered
2888 struct svm_range_list *svms = prange->svms; in svm_range_skip_recover()
2890 spin_lock(&svms->deferred_list_lock); in svm_range_skip_recover()
2891 if (list_empty(&prange->deferred_list) && in svm_range_skip_recover()
2892 list_empty(&prange->child_list)) { in svm_range_skip_recover()
2893 spin_unlock(&svms->deferred_list_lock); in svm_range_skip_recover()
2896 spin_unlock(&svms->deferred_list_lock); in svm_range_skip_recover()
2898 if (prange->work_item.op == SVM_OP_UNMAP_RANGE) { in svm_range_skip_recover()
2900 svms, prange, prange->start, prange->last); in svm_range_skip_recover()
2903 if (prange->work_item.op == SVM_OP_ADD_RANGE_AND_MAP || in svm_range_skip_recover()
2904 prange->work_item.op == SVM_OP_ADD_RANGE) { in svm_range_skip_recover()
2906 svms, prange, prange->start, prange->last); in svm_range_skip_recover()
2932 * or fault cannot recover because GPU no access on the range in svm_range_count_fault()
2936 WRITE_ONCE(pdd->faults, pdd->faults + 1); in svm_range_count_fault()
2948 vma->vm_flags); in svm_fault_allowed()
2949 return (vma->vm_flags & requested) == requested; in svm_fault_allowed()
2973 return -EFAULT; in svm_range_restore_pages()
2981 svms = &p->svms; in svm_range_restore_pages()
2985 if (atomic_read(&svms->drain_pagefaults)) { in svm_range_restore_pages()
2995 r = -EFAULT; in svm_range_restore_pages()
3001 r = -EFAULT; in svm_range_restore_pages()
3005 if (!p->xnack_enabled) { in svm_range_restore_pages()
3007 r = -EFAULT; in svm_range_restore_pages()
3011 /* p->lead_thread is available as kfd_process_wq_release flush the work in svm_range_restore_pages()
3014 mm = get_task_mm(p->lead_thread); in svm_range_restore_pages()
3023 mutex_lock(&svms->lock); in svm_range_restore_pages()
3025 /* check if this page fault time stamp is before svms->checkpoint_ts */ in svm_range_restore_pages()
3026 if (svms->checkpoint_ts[gpuidx] != 0) { in svm_range_restore_pages()
3027 if (amdgpu_ih_ts_after(ts, svms->checkpoint_ts[gpuidx])) { in svm_range_restore_pages()
3029 r = -EAGAIN; in svm_range_restore_pages()
3032 /* ts is after svms->checkpoint_ts now, reset svms->checkpoint_ts in svm_range_restore_pages()
3035 svms->checkpoint_ts[gpuidx] = 0; in svm_range_restore_pages()
3048 mutex_unlock(&svms->lock); in svm_range_restore_pages()
3059 r = -EFAULT; in svm_range_restore_pages()
3066 mutex_lock(&prange->migrate_mutex); in svm_range_restore_pages()
3069 amdgpu_gmc_filter_faults_remove(node->adev, addr, pasid); in svm_range_restore_pages()
3075 if (ktime_before(timestamp, ktime_add_ns(prange->validate_timestamp, in svm_range_restore_pages()
3078 svms, prange->start, prange->last); in svm_range_restore_pages()
3096 r = -EPERM; in svm_range_restore_pages()
3101 if (best_loc == -1) { in svm_range_restore_pages()
3103 svms, prange->start, prange->last); in svm_range_restore_pages()
3104 r = -EACCES; in svm_range_restore_pages()
3109 svms, prange->start, prange->last, best_loc, in svm_range_restore_pages()
3110 prange->actual_loc); in svm_range_restore_pages()
3112 kfd_smi_event_page_fault_start(node, p->lead_thread->pid, addr, in svm_range_restore_pages()
3115 /* Align migration range start and size to granularity size */ in svm_range_restore_pages()
3116 size = 1UL << prange->granularity; in svm_range_restore_pages()
3117 start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start); in svm_range_restore_pages()
3118 last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last); in svm_range_restore_pages()
3119 if (prange->actual_loc != 0 || best_loc != 0) { in svm_range_restore_pages()
3129 if (prange->actual_loc && prange->actual_loc != best_loc) in svm_range_restore_pages()
3155 kfd_smi_event_page_fault_end(node, p->lead_thread->pid, addr, in svm_range_restore_pages()
3159 mutex_unlock(&prange->migrate_mutex); in svm_range_restore_pages()
3161 mutex_unlock(&svms->lock); in svm_range_restore_pages()
3164 if (r != -EAGAIN) in svm_range_restore_pages()
3171 if (r == -EAGAIN) { in svm_range_restore_pages()
3173 amdgpu_gmc_filter_faults_remove(node->adev, addr, pasid); in svm_range_restore_pages()
3187 pr_debug("switching xnack from %d to %d\n", p->xnack_enabled, xnack_enabled); in svm_range_switch_xnack_reserve_mem()
3189 mutex_lock(&p->svms.lock); in svm_range_switch_xnack_reserve_mem()
3191 list_for_each_entry(prange, &p->svms.list, list) { in svm_range_switch_xnack_reserve_mem()
3193 list_for_each_entry(pchild, &prange->child_list, child_list) { in svm_range_switch_xnack_reserve_mem()
3194 size = (pchild->last - pchild->start + 1) << PAGE_SHIFT; in svm_range_switch_xnack_reserve_mem()
3207 size = (prange->last - prange->start + 1) << PAGE_SHIFT; in svm_range_switch_xnack_reserve_mem()
3231 p->xnack_enabled = xnack_enabled; in svm_range_switch_xnack_reserve_mem()
3233 mutex_unlock(&p->svms.lock); in svm_range_switch_xnack_reserve_mem()
3242 pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms); in svm_range_list_fini()
3244 cancel_delayed_work_sync(&p->svms.restore_work); in svm_range_list_fini()
3247 flush_work(&p->svms.deferred_list_work); in svm_range_list_fini()
3254 atomic_set(&p->svms.drain_pagefaults, 1); in svm_range_list_fini()
3255 svm_range_drain_retry_fault(&p->svms); in svm_range_list_fini()
3257 list_for_each_entry_safe(prange, next, &p->svms.list, list) { in svm_range_list_fini()
3263 mutex_destroy(&p->svms.lock); in svm_range_list_fini()
3265 pr_debug("pasid 0x%x svms 0x%p done\n", p->pasid, &p->svms); in svm_range_list_fini()
3270 struct svm_range_list *svms = &p->svms; in svm_range_list_init()
3273 svms->objects = RB_ROOT_CACHED; in svm_range_list_init()
3274 mutex_init(&svms->lock); in svm_range_list_init()
3275 INIT_LIST_HEAD(&svms->list); in svm_range_list_init()
3276 atomic_set(&svms->evicted_ranges, 0); in svm_range_list_init()
3277 atomic_set(&svms->drain_pagefaults, 0); in svm_range_list_init()
3278 INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work); in svm_range_list_init()
3279 INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work); in svm_range_list_init()
3280 INIT_LIST_HEAD(&svms->deferred_range_list); in svm_range_list_init()
3281 INIT_LIST_HEAD(&svms->criu_svm_metadata_list); in svm_range_list_init()
3282 spin_lock_init(&svms->deferred_list_lock); in svm_range_list_init()
3284 for (i = 0; i < p->n_pdds; i++) in svm_range_list_init()
3285 if (KFD_IS_SVM_API_SUPPORTED(p->pdds[i]->dev->adev)) in svm_range_list_init()
3286 bitmap_set(svms->bitmap_supported, i, 1); in svm_range_list_init()
3288 /* Value of default granularity cannot exceed 0x1B, the in svm_range_list_init()
3289 * number of pages supported by a 4-level paging table in svm_range_list_init()
3291 svms->default_granularity = min_t(u8, amdgpu_svm_default_granularity, 0x1B); in svm_range_list_init()
3292 pr_debug("Default SVM Granularity to use: %d\n", svms->default_granularity); in svm_range_list_init()
3298 * svm_range_check_vm - check if virtual address range mapped already
3311 * Return 0 - OK, if the range is not mapped.
3313 * -EADDRINUSE - if address is mapped already by kfd_ioctl_alloc_memory_of_gpu
3314 * -ERESTARTSYS - A wait for the buffer to become unreserved was interrupted by
3315 * a signal. Release all buffer reservations and return to user-space.
3326 for (i = 0; i < p->n_pdds; i++) { in svm_range_check_vm()
3329 if (!p->pdds[i]->drm_priv) in svm_range_check_vm()
3332 vm = drm_priv_to_vm(p->pdds[i]->drm_priv); in svm_range_check_vm()
3333 r = amdgpu_bo_reserve(vm->root.bo, false); in svm_range_check_vm()
3337 node = interval_tree_iter_first(&vm->va, start, last); in svm_range_check_vm()
3344 *bo_s = mapping->start; in svm_range_check_vm()
3345 *bo_l = mapping->last; in svm_range_check_vm()
3347 amdgpu_bo_unreserve(vm->root.bo); in svm_range_check_vm()
3348 return -EADDRINUSE; in svm_range_check_vm()
3350 amdgpu_bo_unreserve(vm->root.bo); in svm_range_check_vm()
3357 * svm_range_is_valid - check if virtual address range is valid
3367 * 0 - OK, otherwise error code
3380 vma = vma_lookup(p->mm, start); in svm_range_is_valid()
3381 if (!vma || (vma->vm_flags & device_vma)) in svm_range_is_valid()
3382 return -EFAULT; in svm_range_is_valid()
3383 start = min(end, vma->vm_end); in svm_range_is_valid()
3386 return svm_range_check_vm(p, start_unchg, (end - 1) >> PAGE_SHIFT, NULL, in svm_range_is_valid()
3391 * svm_range_best_prefetch_location - decide the best prefetch location
3398 * If range is ACCESS or ACCESS_IN_PLACE by mGPUs, only if mGPU connection on
3401 * mapping VRAM of other GPUs even with large-BAR PCIe connection.
3405 * prefetch_loc, other GPU access will generate vm fault and trigger migration.
3420 uint32_t best_loc = prange->prefetch_loc; in svm_range_best_prefetch_location()
3426 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_best_prefetch_location()
3438 if (bo_node->adev->flags & AMD_IS_APU) { in svm_range_best_prefetch_location()
3443 if (p->xnack_enabled) in svm_range_best_prefetch_location()
3444 bitmap_copy(bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_best_prefetch_location()
3446 bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip, in svm_range_best_prefetch_location()
3456 if (pdd->dev->adev == bo_node->adev) in svm_range_best_prefetch_location()
3459 if (!svm_nodes_in_same_hive(pdd->dev, bo_node)) { in svm_range_best_prefetch_location()
3467 p->xnack_enabled, &p->svms, prange->start, prange->last, in svm_range_best_prefetch_location()
3473 /* svm_range_trigger_migration - start page migration if prefetch loc changed
3488 * a. svm_range_validate_vram takes prange->migrate_mutex
3495 * 0 - OK, otherwise - error code of migration
3507 /* when best_loc is a gpu node and same as prange->actual_loc in svm_range_trigger_migration()
3508 * we still need do migration as prange->actual_loc !=0 does in svm_range_trigger_migration()
3513 (best_loc == 0 && prange->actual_loc == 0)) in svm_range_trigger_migration()
3517 r = svm_migrate_vram_to_ram(prange, mm, prange->start, prange->last, in svm_range_trigger_migration()
3523 r = svm_migrate_to_vram(prange, best_loc, prange->start, prange->last, in svm_range_trigger_migration()
3532 /* Dereferencing fence->svm_bo is safe here because the fence hasn't in svm_range_schedule_evict_svm_bo()
3533 * signaled yet and we're under the protection of the fence->lock. in svm_range_schedule_evict_svm_bo()
3539 if (svm_bo_ref_unless_zero(fence->svm_bo)) { in svm_range_schedule_evict_svm_bo()
3540 WRITE_ONCE(fence->svm_bo->evicting, 1); in svm_range_schedule_evict_svm_bo()
3541 schedule_work(&fence->svm_bo->eviction_work); in svm_range_schedule_evict_svm_bo()
3555 if (mmget_not_zero(svm_bo->eviction_fence->mm)) { in svm_range_evict_svm_bo_worker()
3556 mm = svm_bo->eviction_fence->mm; in svm_range_evict_svm_bo_worker()
3563 spin_lock(&svm_bo->list_lock); in svm_range_evict_svm_bo_worker()
3564 while (!list_empty(&svm_bo->range_list) && !r) { in svm_range_evict_svm_bo_worker()
3566 list_first_entry(&svm_bo->range_list, in svm_range_evict_svm_bo_worker()
3570 list_del_init(&prange->svm_bo_list); in svm_range_evict_svm_bo_worker()
3571 spin_unlock(&svm_bo->list_lock); in svm_range_evict_svm_bo_worker()
3573 pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, in svm_range_evict_svm_bo_worker()
3574 prange->start, prange->last); in svm_range_evict_svm_bo_worker()
3576 mutex_lock(&prange->migrate_mutex); in svm_range_evict_svm_bo_worker()
3579 * after that prange->actual_loc should be zero in svm_range_evict_svm_bo_worker()
3582 prange->start, prange->last, in svm_range_evict_svm_bo_worker()
3584 } while (!r && prange->actual_loc && --retries); in svm_range_evict_svm_bo_worker()
3586 if (!r && prange->actual_loc) in svm_range_evict_svm_bo_worker()
3589 if (!prange->actual_loc) { in svm_range_evict_svm_bo_worker()
3590 mutex_lock(&prange->lock); in svm_range_evict_svm_bo_worker()
3591 prange->svm_bo = NULL; in svm_range_evict_svm_bo_worker()
3592 mutex_unlock(&prange->lock); in svm_range_evict_svm_bo_worker()
3594 mutex_unlock(&prange->migrate_mutex); in svm_range_evict_svm_bo_worker()
3596 spin_lock(&svm_bo->list_lock); in svm_range_evict_svm_bo_worker()
3598 spin_unlock(&svm_bo->list_lock); in svm_range_evict_svm_bo_worker()
3602 dma_fence_signal(&svm_bo->eviction_fence->base); in svm_range_evict_svm_bo_worker()
3607 WARN_ONCE(!r && kref_read(&svm_bo->kref) != 1, "This was not the last reference\n"); in svm_range_evict_svm_bo_worker()
3616 struct amdkfd_process_info *process_info = p->kgd_process_info; in svm_range_set_attr()
3629 p->pasid, &p->svms, start, start + size - 1, size); in svm_range_set_attr()
3635 svms = &p->svms; in svm_range_set_attr()
3637 mutex_lock(&process_info->lock); in svm_range_set_attr()
3648 mutex_lock(&svms->lock); in svm_range_set_attr()
3654 mutex_unlock(&svms->lock); in svm_range_set_attr()
3665 /* TODO: unmap ranges from GPU that lost access */ in svm_range_set_attr()
3669 prange->svms, prange, prange->start, in svm_range_set_attr()
3670 prange->last); in svm_range_set_attr()
3685 mutex_lock(&prange->migrate_mutex); in svm_range_set_attr()
3691 if (migrated && (!p->xnack_enabled || in svm_range_set_attr()
3692 (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) && in svm_range_set_attr()
3693 prange->mapped_to_gpu) { in svm_range_set_attr()
3695 mutex_unlock(&prange->migrate_mutex); in svm_range_set_attr()
3700 mutex_unlock(&prange->migrate_mutex); in svm_range_set_attr()
3704 flush_tlb = !migrated && update_mapping && prange->mapped_to_gpu; in svm_range_set_attr()
3706 r = svm_range_validate_and_map(mm, prange->start, prange->last, prange, in svm_range_set_attr()
3712 mutex_unlock(&prange->migrate_mutex); in svm_range_set_attr()
3719 prange, prange->start, prange->last); in svm_range_set_attr()
3720 mutex_lock(&prange->migrate_mutex); in svm_range_set_attr()
3721 r = svm_range_validate_and_map(mm, prange->start, prange->last, prange, in svm_range_set_attr()
3722 MAX_GPU_INSTANCE, true, true, prange->mapped_to_gpu); in svm_range_set_attr()
3725 mutex_unlock(&prange->migrate_mutex); in svm_range_set_attr()
3732 mutex_unlock(&svms->lock); in svm_range_set_attr()
3735 mutex_unlock(&process_info->lock); in svm_range_set_attr()
3737 pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] done, r=%d\n", p->pasid, in svm_range_set_attr()
3738 &p->svms, start, start + size - 1, r); in svm_range_set_attr()
3755 uint64_t last = start + size - 1UL; in svm_range_get_attr()
3756 uint8_t granularity = 0xff; in svm_range_get_attr() local
3768 pr_debug("svms 0x%p [0x%llx 0x%llx] nattr 0x%x\n", &p->svms, start, in svm_range_get_attr()
3769 start + size - 1, nattr); in svm_range_get_attr()
3777 flush_work(&p->svms.deferred_list_work); in svm_range_get_attr()
3810 return -EINVAL; in svm_range_get_attr()
3814 svms = &p->svms; in svm_range_get_attr()
3816 mutex_lock(&svms->lock); in svm_range_get_attr()
3818 node = interval_tree_iter_first(&svms->objects, start, last); in svm_range_get_attr()
3822 &granularity, &flags_and); in svm_range_get_attr()
3824 if (p->xnack_enabled) in svm_range_get_attr()
3825 bitmap_copy(bitmap_access, svms->bitmap_supported, in svm_range_get_attr()
3832 bitmap_copy(bitmap_access, svms->bitmap_supported, MAX_GPU_INSTANCE); in svm_range_get_attr()
3833 bitmap_copy(bitmap_aip, svms->bitmap_supported, MAX_GPU_INSTANCE); in svm_range_get_attr()
3842 if (prange->preferred_loc == in svm_range_get_attr()
3845 location != prange->preferred_loc)) { in svm_range_get_attr()
3849 location = prange->preferred_loc; in svm_range_get_attr()
3853 if (prange->prefetch_loc == in svm_range_get_attr()
3856 prefetch_loc != prange->prefetch_loc)) { in svm_range_get_attr()
3860 prefetch_loc = prange->prefetch_loc; in svm_range_get_attr()
3865 prange->bitmap_access, MAX_GPU_INSTANCE); in svm_range_get_attr()
3867 prange->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_get_attr()
3870 flags_and &= prange->flags; in svm_range_get_attr()
3871 flags_or |= prange->flags; in svm_range_get_attr()
3874 if (get_granularity && prange->granularity < granularity) in svm_range_get_attr()
3875 granularity = prange->granularity; in svm_range_get_attr()
3880 mutex_unlock(&svms->lock); in svm_range_get_attr()
3895 return -EINVAL; in svm_range_get_attr()
3912 attrs[i].value = (uint32_t)granularity; in svm_range_get_attr()
3925 struct svm_range_list *svms = &p->svms; in kfd_criu_resume_svm()
3932 if (list_empty(&svms->criu_svm_metadata_list)) { in kfd_criu_resume_svm()
3937 mm = get_task_mm(p->lead_thread); in kfd_criu_resume_svm()
3940 return -ESRCH; in kfd_criu_resume_svm()
3943 num_attrs = nattr_common + (nattr_accessibility * p->n_pdds); in kfd_criu_resume_svm()
3946 list_for_each_entry(criu_svm_md, &svms->criu_svm_metadata_list, list) { in kfd_criu_resume_svm()
3948 i, criu_svm_md->data.start_addr, criu_svm_md->data.size); in kfd_criu_resume_svm()
3951 pr_debug("\ncriu_svm_md[%d]->attrs[%d].type : 0x%x\ncriu_svm_md[%d]->attrs[%d].value : 0x%x\n", in kfd_criu_resume_svm()
3952 i, j, criu_svm_md->data.attrs[j].type, in kfd_criu_resume_svm()
3953 i, j, criu_svm_md->data.attrs[j].value); in kfd_criu_resume_svm()
3954 switch (criu_svm_md->data.attrs[j].type) { in kfd_criu_resume_svm()
3966 if (criu_svm_md->data.attrs[j].value == in kfd_criu_resume_svm()
3968 criu_svm_md->data.attrs[j].type = in kfd_criu_resume_svm()
3970 criu_svm_md->data.attrs[j].value = 0; in kfd_criu_resume_svm()
3974 set_flags = criu_svm_md->data.attrs[j].value; in kfd_criu_resume_svm()
3990 ret = -ENOMEM; in kfd_criu_resume_svm()
3995 memcpy(set_attr, criu_svm_md->data.attrs, num_attrs * in kfd_criu_resume_svm()
4000 ret = svm_range_set_attr(p, mm, criu_svm_md->data.start_addr, in kfd_criu_resume_svm()
4001 criu_svm_md->data.size, num_attrs + 1, in kfd_criu_resume_svm()
4012 list_for_each_entry_safe(criu_svm_md, next, &svms->criu_svm_metadata_list, list) { in kfd_criu_resume_svm()
4014 criu_svm_md->data.start_addr); in kfd_criu_resume_svm()
4031 struct svm_range_list *svms = &p->svms; in kfd_criu_restore_svm()
4035 num_devices = p->n_pdds; in kfd_criu_restore_svm()
4051 return -ENOMEM; in kfd_criu_restore_svm()
4054 ret = -EINVAL; in kfd_criu_restore_svm()
4058 ret = copy_from_user(&criu_svm_md->data, user_priv_ptr + *priv_data_offset, in kfd_criu_restore_svm()
4061 ret = -EFAULT; in kfd_criu_restore_svm()
4066 list_add_tail(&criu_svm_md->list, &svms->criu_svm_metadata_list); in kfd_criu_restore_svm()
4081 int num_devices = p->n_pdds; in svm_range_get_info()
4088 svms = &p->svms; in svm_range_get_info()
4090 return -EINVAL; in svm_range_get_info()
4092 mutex_lock(&svms->lock); in svm_range_get_info()
4093 list_for_each_entry(prange, &svms->list, list) { in svm_range_get_info()
4095 prange, prange->start, prange->npages, in svm_range_get_info()
4096 prange->start + prange->npages - 1); in svm_range_get_info()
4099 mutex_unlock(&svms->lock); in svm_range_get_info()
4144 int num_devices = p->n_pdds; in kfd_criu_checkpoint_svm()
4148 svms = &p->svms; in kfd_criu_checkpoint_svm()
4150 return -EINVAL; in kfd_criu_checkpoint_svm()
4152 mm = get_task_mm(p->lead_thread); in kfd_criu_checkpoint_svm()
4155 return -ESRCH; in kfd_criu_checkpoint_svm()
4163 ret = -ENOMEM; in kfd_criu_checkpoint_svm()
4173 struct kfd_process_device *pdd = p->pdds[index]; in kfd_criu_checkpoint_svm()
4177 query_attr[index + nattr_common].value = pdd->user_gpu_id; in kfd_criu_checkpoint_svm()
4184 ret = -ENOMEM; in kfd_criu_checkpoint_svm()
4189 list_for_each_entry(prange, &svms->list, list) { in kfd_criu_checkpoint_svm()
4191 svm_priv->object_type = KFD_CRIU_OBJECT_TYPE_SVM_RANGE; in kfd_criu_checkpoint_svm()
4192 svm_priv->start_addr = prange->start; in kfd_criu_checkpoint_svm()
4193 svm_priv->size = prange->npages; in kfd_criu_checkpoint_svm()
4194 memcpy(&svm_priv->attrs, query_attr, query_attr_size); in kfd_criu_checkpoint_svm()
4196 prange, prange->start, prange->npages, in kfd_criu_checkpoint_svm()
4197 prange->start + prange->npages - 1, in kfd_criu_checkpoint_svm()
4198 prange->npages * PAGE_SIZE); in kfd_criu_checkpoint_svm()
4200 ret = svm_range_get_attr(p, mm, svm_priv->start_addr, in kfd_criu_checkpoint_svm()
4201 svm_priv->size, in kfd_criu_checkpoint_svm()
4203 svm_priv->attrs); in kfd_criu_checkpoint_svm()
4212 ret = -EFAULT; in kfd_criu_checkpoint_svm()
4234 struct mm_struct *mm = current->mm; in svm_ioctl()