Lines Matching +full:smem +full:- +full:part
1 // SPDX-License-Identifier: MIT
6 #include <linux/dma-fence-array.h>
29 /** @children: Array of page-table child nodes */
31 /** @staging: Array of page-table staging nodes */
36 #define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr))
37 #define xe_pt_addr(__xe_pt) ((__xe_pt)->addr)
46 #define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1)
56 return container_of(pt_dir->staging[index], struct xe_pt, base); in xe_pt_entry_staging()
63 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; in __xe_pt_empty_pte()
64 u8 id = tile->id; in __xe_pt_empty_pte()
70 return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo, in __xe_pt_empty_pte()
73 return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) | in __xe_pt_empty_pte()
79 if (pt->level) in xe_pt_free()
86 * xe_pt_create() - Create a page-table.
89 * @level: The page-table level.
92 * create the corresponding page-table bo, but don't initialize it. If the
93 * level is grater than zero, then it's assumed to be a directory page-
110 pt = (dir) ? &dir->pt : NULL; in xe_pt_create()
115 return ERR_PTR(-ENOMEM); in xe_pt_create()
117 pt->level = level; in xe_pt_create()
118 bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K, in xe_pt_create()
129 pt->bo = bo; in xe_pt_create()
130 pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL; in xe_pt_create()
131 pt->base.staging = level ? as_xe_pt_dir(pt)->staging : NULL; in xe_pt_create()
133 if (vm->xef) in xe_pt_create()
134 xe_drm_client_add_bo(vm->xef->client, pt->bo); in xe_pt_create()
146 * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero
152 * Populate the page-table bo of @pt with entries pointing into the tile's
153 * scratch page-table tree if any. Otherwise populate with zeros.
158 struct iosys_map *map = &pt->bo->vmap; in xe_pt_populate_empty()
167 xe_map_memset(vm->xe, map, 0, 0, SZ_4K); in xe_pt_populate_empty()
169 empty = __xe_pt_empty_pte(tile, vm, pt->level); in xe_pt_populate_empty()
171 xe_pt_write(vm->xe, map, i, empty); in xe_pt_populate_empty()
176 * xe_pt_shift() - Return the ilog2 value of the size of the address range of
177 * a page-table at a certain level.
180 * Return: The ilog2 value of the size of the address range of a page-table
189 * xe_pt_destroy() - Destroy a page-table tree.
190 * @pt: The root of the page-table tree to destroy.
195 * Puts the page-table bo, recursively calls xe_pt_destroy on all children
205 XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list)); in xe_pt_destroy()
206 xe_bo_unpin(pt->bo); in xe_pt_destroy()
207 xe_bo_put_deferred(pt->bo, deferred); in xe_pt_destroy()
209 if (pt->level > 0 && pt->num_live) { in xe_pt_destroy()
224 * Below we use the term "page-table" for both page-directories, containing
225 * pointers to lower level page-directories or page-tables, and level 0
226 * page-tables that contain only page-table-entries pointing to memory pages.
228 * When inserting an address range in an already existing page-table tree
229 * there will typically be a set of page-tables that are shared with other
231 * The set of shared page-tables can be at most two per level,
233 * page-tables may still be in use by the gpu for other mappings. Therefore
236 * data, (subtrees for the cpu and page-table-entries for the gpu) is then
237 * added in a separate commit step. CPU-data is committed while still under the
241 * For non-shared page-tables (and, in fact, for shared ones that aren't
242 * existing at the time of staging), we add the data in-place without the
243 * special update structures. This private part of the page-table tree will
244 * remain disconnected from the vm page-table tree until data is committed to
253 /** @preexisting: Whether the parent was pre-existing or allocated */
283 /** @va_curs_start: The Virtual address corresponding to @curs->start */
312 struct xe_pt_update *upd = &wupd->updates[parent->level]; in xe_pt_new_shared()
318 * new parent and page-directory, the old one is complete, and in xe_pt_new_shared()
320 * @wupd->entries in xe_pt_new_shared()
322 if (likely(upd->parent == parent)) in xe_pt_new_shared()
325 upd->parent = parent; in xe_pt_new_shared()
326 upd->preexisting = true; in xe_pt_new_shared()
328 if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1) in xe_pt_new_shared()
329 return -EINVAL; in xe_pt_new_shared()
331 entry = wupd->entries + wupd->num_used_entries++; in xe_pt_new_shared()
332 upd->update = entry; in xe_pt_new_shared()
333 entry->ofs = offset; in xe_pt_new_shared()
334 entry->pt_bo = parent->bo; in xe_pt_new_shared()
335 entry->pt = parent; in xe_pt_new_shared()
336 entry->flags = 0; in xe_pt_new_shared()
337 entry->qwords = 0; in xe_pt_new_shared()
338 entry->pt_bo->update_index = -1; in xe_pt_new_shared()
341 entry->pt_entries = kmalloc_array(XE_PDES, in xe_pt_new_shared()
342 sizeof(*entry->pt_entries), in xe_pt_new_shared()
344 if (!entry->pt_entries) in xe_pt_new_shared()
345 return -ENOMEM; in xe_pt_new_shared()
354 * non-pre-existing pagetable with leaf ptes.
360 struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level]; in xe_pt_insert_entry()
362 &xe_walk->wupd.updates[xe_child->level] : NULL; in xe_pt_insert_entry()
365 ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true); in xe_pt_insert_entry()
374 child_upd->update = NULL; in xe_pt_insert_entry()
375 child_upd->parent = xe_child; in xe_pt_insert_entry()
376 child_upd->preexisting = false; in xe_pt_insert_entry()
379 if (likely(!upd->preexisting)) { in xe_pt_insert_entry()
380 /* Continue building a non-connected subtree. */ in xe_pt_insert_entry()
381 struct iosys_map *map = &parent->bo->vmap; in xe_pt_insert_entry()
384 parent->base.children[offset] = &xe_child->base; in xe_pt_insert_entry()
385 parent->base.staging[offset] = &xe_child->base; in xe_pt_insert_entry()
388 xe_pt_write(xe_walk->vm->xe, map, offset, pte); in xe_pt_insert_entry()
389 parent->num_live++; in xe_pt_insert_entry()
393 struct xe_vm_pgtable_update *entry = upd->update; in xe_pt_insert_entry()
395 idx = offset - entry->ofs; in xe_pt_insert_entry()
396 entry->pt_entries[idx].pt = xe_child; in xe_pt_insert_entry()
397 entry->pt_entries[idx].pte = pte; in xe_pt_insert_entry()
398 entry->qwords++; in xe_pt_insert_entry()
413 if (!xe_pt_covers(addr, next, level, &xe_walk->base)) in xe_pt_hugepte_possible()
417 if (next - xe_walk->va_curs_start > xe_walk->curs->size) in xe_pt_hugepte_possible()
421 if (xe_vma_is_null(xe_walk->vma)) in xe_pt_hugepte_possible()
425 size = next - addr; in xe_pt_hugepte_possible()
426 dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs); in xe_pt_hugepte_possible()
438 struct xe_res_cursor curs = *xe_walk->curs; in xe_pt_scan_64K()
443 if (next > xe_walk->l0_end_addr) in xe_pt_scan_64K()
447 if (xe_vma_is_null(xe_walk->vma)) in xe_pt_scan_64K()
450 xe_res_next(&curs, addr - xe_walk->va_curs_start); in xe_pt_scan_64K()
462 * For non-compact "normal" 4K level-0 pagetables, we want to try to group
463 * addresses together in 64K-contigous regions to add a 64K TLB hint for the
465 * This function determines whether the address is part of such a
473 if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K) in xe_pt_is_pte_ps64K()
476 xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk); in xe_pt_is_pte_ps64K()
477 xe_walk->addr_64K = addr; in xe_pt_is_pte_ps64K()
479 return xe_walk->found_64K; in xe_pt_is_pte_ps64K()
491 u16 pat_index = xe_walk->vma->pat_index; in xe_pt_stage_bind_entry()
493 struct xe_vm *vm = xe_walk->vm; in xe_pt_stage_bind_entry()
501 struct xe_res_cursor *curs = xe_walk->curs; in xe_pt_stage_bind_entry()
502 bool is_null = xe_vma_is_null(xe_walk->vma); in xe_pt_stage_bind_entry()
504 XE_WARN_ON(xe_walk->va_curs_start != addr); in xe_pt_stage_bind_entry()
506 pte = vm->pt_ops->pte_encode_vma(is_null ? 0 : in xe_pt_stage_bind_entry()
507 xe_res_dma(curs) + xe_walk->dma_offset, in xe_pt_stage_bind_entry()
508 xe_walk->vma, pat_index, level); in xe_pt_stage_bind_entry()
509 pte |= xe_walk->default_pte; in xe_pt_stage_bind_entry()
515 if (level == 0 && !xe_parent->is_compact) { in xe_pt_stage_bind_entry()
517 xe_walk->vma->gpuva.flags |= XE_VMA_PTE_64K; in xe_pt_stage_bind_entry()
519 } else if (XE_WARN_ON(xe_walk->needs_64K)) { in xe_pt_stage_bind_entry()
520 return -EINVAL; in xe_pt_stage_bind_entry()
529 xe_res_next(curs, next - addr); in xe_pt_stage_bind_entry()
530 xe_walk->va_curs_start = next; in xe_pt_stage_bind_entry()
531 xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level); in xe_pt_stage_bind_entry()
539 * new page table or -directory, which we do if there is no in xe_pt_stage_bind_entry()
543 walk->shifts = xe_normal_pt_shifts; in xe_pt_stage_bind_entry()
544 xe_walk->l0_end_addr = next; in xe_pt_stage_bind_entry()
547 covers = xe_pt_covers(addr, next, level, &xe_walk->base); in xe_pt_stage_bind_entry()
551 xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1); in xe_pt_stage_bind_entry()
556 round_down(addr, 1ull << walk->shifts[level])); in xe_pt_stage_bind_entry()
559 xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child); in xe_pt_stage_bind_entry()
561 *child = &xe_child->base; in xe_pt_stage_bind_entry()
570 if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 && in xe_pt_stage_bind_entry()
572 walk->shifts = xe_compact_pt_shifts; in xe_pt_stage_bind_entry()
573 xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT; in xe_pt_stage_bind_entry()
575 xe_child->is_compact = true; in xe_pt_stage_bind_entry()
578 pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0, pat_index) | flags; in xe_pt_stage_bind_entry()
592 * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address
600 * This function builds a disconnected page-table tree for a given address
604 * shared page-tables, and non-existing shared and non-shared page-tables
631 .needs_64K = (xe_vma_vm(vma)->flags & XE_VM_FLAG_64K) && is_devmem, in xe_pt_stage_bind()
633 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; in xe_pt_stage_bind()
640 * - Device atomics are expected to function with all allocations. in xe_pt_stage_bind()
643 * - Device atomics are the default behavior when the bo is placed in a single region. in xe_pt_stage_bind()
644 * - In all other cases device atomics will be disabled with AE=0 until an application in xe_pt_stage_bind()
647 if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) { in xe_pt_stage_bind()
652 * If a SMEM+LMEM allocation is backed by SMEM, a device in xe_pt_stage_bind()
667 if (!xe->info.has_device_atomics_on_smem && !is_devmem) in xe_pt_stage_bind()
673 xe_walk.dma_offset = vram_region_gpu_offset(bo->ttm.resource); in xe_pt_stage_bind()
683 xe_res_first_sg(to_userptr_vma(vma)->userptr.sg, 0, in xe_pt_stage_bind()
686 xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma), in xe_pt_stage_bind()
695 ret = xe_pt_walk_range(&pt->base, pt->level, xe_vma_start(vma), in xe_pt_stage_bind()
703 * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a
705 * @addr: The start address within the non-shared pagetable.
706 * @end: The end address within the non-shared pagetable.
707 * @level: The level of the non-shared pagetable.
710 * @offset: Ignored on input, First non-shared entry on output.
711 * @end_offset: Ignored on input, Last non-shared entry + 1 on output.
713 * A non-shared page-table has some entries that belong to the address range
722 * Return: true if there were non-shared entries, false otherwise.
729 u64 size = 1ull << walk->shifts[level]; in xe_pt_nonshared_offsets()
749 (*end_offset)--; in xe_pt_nonshared_offsets()
756 /** @base: The walk base-class */
787 if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset, in xe_pt_zap_ptes_entry()
789 xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap, in xe_pt_zap_ptes_entry()
791 (end_offset - offset) * sizeof(u64)); in xe_pt_zap_ptes_entry()
792 xe_walk->needs_invalidate = true; in xe_pt_zap_ptes_entry()
803 * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range
810 * page-table entrieaso it can either clear the leaf PTEs or
811 * clear the pointers to lower-level page-tables. The caller is required
812 * to hold the necessary locks to ensure neither the page-table connectivity
813 * nor the page-table entries of the range is updated from under us.
828 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; in xe_pt_zap_ptes()
829 u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated); in xe_pt_zap_ptes()
831 if (!(pt_mask & BIT(tile->id))) in xe_pt_zap_ptes()
834 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), in xe_pt_zap_ptes()
846 struct xe_pt_entry *ptes = update->pt_entries; in xe_vm_populate_pgtable()
871 if (pt->level) { in xe_pt_cancel_bind()
874 xe_vma_vm(vma)->flags, NULL); in xe_pt_cancel_bind()
887 lockdep_assert_held(&vm->lock); in xe_pt_commit_prepare_locks_assert()
890 dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv); in xe_pt_commit_prepare_locks_assert()
902 lockdep_assert_held_read(&vm->userptr.notifier_lock); in xe_pt_commit_locks_assert()
917 if (!pt->level) in xe_pt_commit()
925 pt_dir->children[j_] = pt_dir->staging[j_]; in xe_pt_commit()
926 xe_pt_destroy(oldpte, xe_vma_vm(vma)->flags, deferred); in xe_pt_commit()
939 for (i = num_entries - 1; i >= 0; --i) { in xe_pt_abort_bind()
944 pt->num_live -= entries[i].qwords; in xe_pt_abort_bind()
946 if (!pt->level) in xe_pt_abort_bind()
955 pt_dir->staging[j_] = oldpte ? &oldpte->base : 0; in xe_pt_abort_bind()
956 xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL); in xe_pt_abort_bind()
974 pt->num_live += entries[i].qwords; in xe_pt_commit_prepare_bind()
976 if (!pt->level) in xe_pt_commit_prepare_bind()
988 pt_dir->staging[j_] = &newpte->base; in xe_pt_commit_prepare_bind()
1024 vm_dbg(&xe->drm, "%s: %u entries to update\n", bind ? "bind" : "unbind", in xe_vm_dbg_print_entries()
1028 struct xe_pt *xe_pt = entry->pt; in xe_vm_dbg_print_entries()
1029 u64 page_size = 1ull << xe_pt_shift(xe_pt->level); in xe_vm_dbg_print_entries()
1033 xe_assert(xe, !entry->pt->is_compact); in xe_vm_dbg_print_entries()
1034 start = entry->ofs * page_size; in xe_vm_dbg_print_entries()
1035 end = start + page_size * entry->qwords; in xe_vm_dbg_print_entries()
1036 vm_dbg(&xe->drm, in xe_vm_dbg_print_entries()
1038 i, xe_pt->level, entry->ofs, entry->qwords, in xe_vm_dbg_print_entries()
1054 &fence->flags)) in no_in_syncs()
1067 return -ETIME; in job_test_add_deps()
1081 if (bo && !bo->vm) in vma_add_deps()
1082 return job_test_add_deps(job, bo->ttm.base.resv, in vma_add_deps()
1093 switch (op->base.op) { in op_add_deps()
1095 if (!op->map.immediate && xe_vm_in_fault_mode(vm)) in op_add_deps()
1098 err = vma_add_deps(op->map.vma, job); in op_add_deps()
1101 if (op->remap.prev) in op_add_deps()
1102 err = vma_add_deps(op->remap.prev, job); in op_add_deps()
1103 if (!err && op->remap.next) in op_add_deps()
1104 err = vma_add_deps(op->remap.next, job); in op_add_deps()
1109 err = vma_add_deps(gpuva_to_vma(op->base.prefetch.va), job); in op_add_deps()
1112 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_add_deps()
1131 if (!job && !no_in_syncs(vops->syncs, vops->num_syncs)) in xe_pt_vm_dependencies()
1132 return -ETIME; in xe_pt_vm_dependencies()
1134 if (!job && !xe_exec_queue_is_idle(pt_update_ops->q)) in xe_pt_vm_dependencies()
1135 return -ETIME; in xe_pt_vm_dependencies()
1137 if (pt_update_ops->wait_vm_bookkeep || pt_update_ops->wait_vm_kernel) { in xe_pt_vm_dependencies()
1139 pt_update_ops->wait_vm_bookkeep ? in xe_pt_vm_dependencies()
1146 rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start, in xe_pt_vm_dependencies()
1147 pt_update_ops->last); in xe_pt_vm_dependencies()
1149 fence = rtfence->fence; in xe_pt_vm_dependencies()
1157 return -ETIME; in xe_pt_vm_dependencies()
1160 err = drm_sched_job_add_dependency(&job->drm, fence); in xe_pt_vm_dependencies()
1166 pt_update_ops->start, in xe_pt_vm_dependencies()
1167 pt_update_ops->last); in xe_pt_vm_dependencies()
1170 list_for_each_entry(op, &vops->list, link) { in xe_pt_vm_dependencies()
1176 if (!(pt_update_ops->q->flags & EXEC_QUEUE_FLAG_KERNEL)) { in xe_pt_vm_dependencies()
1180 err = xe_exec_queue_last_fence_test_dep(pt_update_ops->q, vm); in xe_pt_vm_dependencies()
1183 for (i = 0; job && !err && i < vops->num_syncs; i++) in xe_pt_vm_dependencies()
1184 err = xe_sync_entry_add_deps(&vops->syncs[i], job); in xe_pt_vm_dependencies()
1191 struct xe_vma_ops *vops = pt_update->vops; in xe_pt_pre_commit()
1192 struct xe_vm *vm = vops->vm; in xe_pt_pre_commit()
1193 struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id]; in xe_pt_pre_commit()
1195 &vops->pt_update_ops[pt_update->tile_id]; in xe_pt_pre_commit()
1197 return xe_pt_vm_dependencies(pt_update->job, vm, pt_update->vops, in xe_pt_pre_commit()
1205 u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2; in xe_pt_userptr_inject_eagain()
1208 if (count++ % divisor == divisor - 1) { in xe_pt_userptr_inject_eagain()
1209 uvma->userptr.divisor = divisor << 1; in xe_pt_userptr_inject_eagain()
1231 lockdep_assert_held_read(&vm->userptr.notifier_lock); in vma_check_userptr()
1240 notifier_seq = uvma->userptr.notifier_seq; in vma_check_userptr()
1242 if (!mmu_interval_read_retry(&uvma->userptr.notifier, in vma_check_userptr()
1247 return -EAGAIN; in vma_check_userptr()
1261 lockdep_assert_held_read(&vm->userptr.notifier_lock); in op_check_userptr()
1263 switch (op->base.op) { in op_check_userptr()
1265 if (!op->map.immediate && xe_vm_in_fault_mode(vm)) in op_check_userptr()
1268 err = vma_check_userptr(vm, op->map.vma, pt_update); in op_check_userptr()
1271 if (op->remap.prev) in op_check_userptr()
1272 err = vma_check_userptr(vm, op->remap.prev, pt_update); in op_check_userptr()
1273 if (!err && op->remap.next) in op_check_userptr()
1274 err = vma_check_userptr(vm, op->remap.next, pt_update); in op_check_userptr()
1279 err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va), in op_check_userptr()
1283 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_check_userptr()
1291 struct xe_vm *vm = pt_update->vops->vm; in xe_pt_userptr_pre_commit()
1292 struct xe_vma_ops *vops = pt_update->vops; in xe_pt_userptr_pre_commit()
1294 &vops->pt_update_ops[pt_update->tile_id]; in xe_pt_userptr_pre_commit()
1302 down_read(&vm->userptr.notifier_lock); in xe_pt_userptr_pre_commit()
1304 list_for_each_entry(op, &vops->list, link) { in xe_pt_userptr_pre_commit()
1307 up_read(&vm->userptr.notifier_lock); in xe_pt_userptr_pre_commit()
1331 struct xe_device *xe = gt_to_xe(ifence->gt); in invalidation_fence_cb()
1333 trace_xe_gt_tlb_invalidation_fence_cb(xe, &ifence->base); in invalidation_fence_cb()
1334 if (!ifence->fence->error) { in invalidation_fence_cb()
1335 queue_work(system_wq, &ifence->work); in invalidation_fence_cb()
1337 ifence->base.base.error = ifence->fence->error; in invalidation_fence_cb()
1338 xe_gt_tlb_invalidation_fence_signal(&ifence->base); in invalidation_fence_cb()
1340 dma_fence_put(ifence->fence); in invalidation_fence_cb()
1347 struct xe_device *xe = gt_to_xe(ifence->gt); in invalidation_fence_work_func()
1349 trace_xe_gt_tlb_invalidation_fence_work_func(xe, &ifence->base); in invalidation_fence_work_func()
1350 xe_gt_tlb_invalidation_range(ifence->gt, &ifence->base, ifence->start, in invalidation_fence_work_func()
1351 ifence->end, ifence->asid); in invalidation_fence_work_func()
1361 trace_xe_gt_tlb_invalidation_fence_create(gt_to_xe(gt), &ifence->base); in invalidation_fence_init()
1363 xe_gt_tlb_invalidation_fence_init(gt, &ifence->base, false); in invalidation_fence_init()
1365 ifence->fence = fence; in invalidation_fence_init()
1366 ifence->gt = gt; in invalidation_fence_init()
1367 ifence->start = start; in invalidation_fence_init()
1368 ifence->end = end; in invalidation_fence_init()
1369 ifence->asid = asid; in invalidation_fence_init()
1371 INIT_WORK(&ifence->work, invalidation_fence_work_func); in invalidation_fence_init()
1372 ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb); in invalidation_fence_init()
1373 if (ret == -ENOENT) { in invalidation_fence_init()
1374 dma_fence_put(ifence->fence); /* Usually dropped in CB */ in invalidation_fence_init()
1375 invalidation_fence_work_func(&ifence->work); in invalidation_fence_init()
1377 dma_fence_put(&ifence->base.base); /* Caller ref */ in invalidation_fence_init()
1378 dma_fence_put(&ifence->base.base); /* Creation ref */ in invalidation_fence_init()
1381 xe_gt_assert(gt, !ret || ret == -ENOENT); in invalidation_fence_init()
1385 /** @base: The pagewalk base-class. */
1402 /* @wupd: Structure to track the page-table updates we're building */
1418 unsigned int shift = walk->shifts[level]; in xe_pt_check_kill()
1422 ((next - addr) >> shift) == child->num_live) { in xe_pt_check_kill()
1423 u64 size = 1ull << walk->shifts[level + 1]; in xe_pt_check_kill()
1427 if (xe_walk->modified_start >= addr) in xe_pt_check_kill()
1428 xe_walk->modified_start = round_down(addr, size); in xe_pt_check_kill()
1429 if (xe_walk->modified_end <= next) in xe_pt_check_kill()
1430 xe_walk->modified_end = round_up(next, size); in xe_pt_check_kill()
1449 xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk); in xe_pt_stage_unbind_entry()
1465 u64 size = 1ull << walk->shifts[--level]; in xe_pt_stage_unbind_post_descend()
1469 addr = xe_walk->modified_start; in xe_pt_stage_unbind_post_descend()
1471 next = xe_walk->modified_end; in xe_pt_stage_unbind_post_descend()
1482 err = xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true); in xe_pt_stage_unbind_post_descend()
1486 xe_walk->wupd.updates[level].update->qwords = end_offset - offset; in xe_pt_stage_unbind_post_descend()
1497 * xe_pt_stage_unbind() - Build page-table update structures for an unbind
1501 * @entries: Caller-provided storage for the update structures.
1503 * Builds page-table update structures for an unbind operation. The function
1504 * will attempt to remove all page-tables that we're the only user
1506 * same critical section that blocks racing binds to the same page-table tree.
1525 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id]; in xe_pt_stage_unbind()
1527 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), in xe_pt_stage_unbind()
1539 struct xe_vm *vm = pt_update->vops->vm; in xe_migrate_clear_pgtable_callback()
1540 u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level); in xe_migrate_clear_pgtable_callback()
1543 if (map && map->is_iomem) in xe_migrate_clear_pgtable_callback()
1548 memset64(map->vaddr + qword_ofs * sizeof(u64), empty, in xe_migrate_clear_pgtable_callback()
1562 for (i = num_entries - 1; i >= 0; --i) { in xe_pt_abort_unbind()
1564 struct xe_pt *pt = entry->pt; in xe_pt_abort_unbind()
1567 pt->num_live += entry->qwords; in xe_pt_abort_unbind()
1569 if (!pt->level) in xe_pt_abort_unbind()
1572 for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) in xe_pt_abort_unbind()
1573 pt_dir->staging[j] = in xe_pt_abort_unbind()
1574 entries[i].pt_entries[j - entry->ofs].pt ? in xe_pt_abort_unbind()
1575 &entries[i].pt_entries[j - entry->ofs].pt->base : NULL; in xe_pt_abort_unbind()
1590 struct xe_pt *pt = entry->pt; in xe_pt_commit_prepare_unbind()
1593 pt->num_live -= entry->qwords; in xe_pt_commit_prepare_unbind()
1594 if (!pt->level) in xe_pt_commit_prepare_unbind()
1598 for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) { in xe_pt_commit_prepare_unbind()
1599 entry->pt_entries[j - entry->ofs].pt = in xe_pt_commit_prepare_unbind()
1601 pt_dir->staging[j] = NULL; in xe_pt_commit_prepare_unbind()
1610 u32 current_op = pt_update_ops->current_op; in xe_pt_update_ops_rfence_interval()
1611 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; in xe_pt_update_ops_rfence_interval()
1615 for (i = 0; i < pt_op->num_entries; i++) { in xe_pt_update_ops_rfence_interval()
1616 const struct xe_vm_pgtable_update *entry = &pt_op->entries[i]; in xe_pt_update_ops_rfence_interval()
1618 if (entry->pt->level > level) in xe_pt_update_ops_rfence_interval()
1619 level = entry->pt->level; in xe_pt_update_ops_rfence_interval()
1622 /* Greedy (non-optimal) calculation but simple */ in xe_pt_update_ops_rfence_interval()
1624 last = ALIGN(xe_vma_end(vma), 0x1ull << xe_pt_shift(level)) - 1; in xe_pt_update_ops_rfence_interval()
1626 if (start < pt_update_ops->start) in xe_pt_update_ops_rfence_interval()
1627 pt_update_ops->start = start; in xe_pt_update_ops_rfence_interval()
1628 if (last > pt_update_ops->last) in xe_pt_update_ops_rfence_interval()
1629 pt_update_ops->last = last; in xe_pt_update_ops_rfence_interval()
1634 int shift = xe_device_get_root_tile(xe)->media_gt ? 1 : 0; in vma_reserve_fences()
1636 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) in vma_reserve_fences()
1637 return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, in vma_reserve_fences()
1638 xe->info.tile_count << shift); in vma_reserve_fences()
1647 u32 current_op = pt_update_ops->current_op; in bind_op_prepare()
1648 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; in bind_op_prepare()
1653 vm_dbg(&xe_vma_vm(vma)->xe->drm, in bind_op_prepare()
1655 xe_vma_start(vma), xe_vma_end(vma) - 1); in bind_op_prepare()
1657 pt_op->vma = NULL; in bind_op_prepare()
1658 pt_op->bind = true; in bind_op_prepare()
1659 pt_op->rebind = BIT(tile->id) & vma->tile_present; in bind_op_prepare()
1665 err = xe_pt_prepare_bind(tile, vma, pt_op->entries, in bind_op_prepare()
1666 &pt_op->num_entries); in bind_op_prepare()
1668 xe_tile_assert(tile, pt_op->num_entries <= in bind_op_prepare()
1669 ARRAY_SIZE(pt_op->entries)); in bind_op_prepare()
1670 xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, in bind_op_prepare()
1671 pt_op->num_entries, true); in bind_op_prepare()
1674 ++pt_update_ops->current_op; in bind_op_prepare()
1675 pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma); in bind_op_prepare()
1680 * automatically when the context is re-enabled by the rebind worker, in bind_op_prepare()
1686 * non-faulting LR, in particular on user-space batch buffer chaining, in bind_op_prepare()
1689 if ((!pt_op->rebind && xe_vm_has_scratch(vm) && in bind_op_prepare()
1691 pt_update_ops->needs_invalidation = true; in bind_op_prepare()
1692 else if (pt_op->rebind && !xe_vm_in_lr_mode(vm)) in bind_op_prepare()
1694 vm->tlb_flush_seqno++; in bind_op_prepare()
1696 vma->tile_staged |= BIT(tile->id); in bind_op_prepare()
1697 pt_op->vma = vma; in bind_op_prepare()
1698 xe_pt_commit_prepare_bind(vma, pt_op->entries, in bind_op_prepare()
1699 pt_op->num_entries, pt_op->rebind); in bind_op_prepare()
1701 xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries); in bind_op_prepare()
1711 u32 current_op = pt_update_ops->current_op; in unbind_op_prepare()
1712 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; in unbind_op_prepare()
1715 if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id))) in unbind_op_prepare()
1720 vm_dbg(&xe_vma_vm(vma)->xe->drm, in unbind_op_prepare()
1722 xe_vma_start(vma), xe_vma_end(vma) - 1); in unbind_op_prepare()
1729 mmu_interval_read_begin(&to_userptr_vma(vma)->userptr.notifier); in unbind_op_prepare()
1731 pt_op->vma = vma; in unbind_op_prepare()
1732 pt_op->bind = false; in unbind_op_prepare()
1733 pt_op->rebind = false; in unbind_op_prepare()
1739 pt_op->num_entries = xe_pt_stage_unbind(tile, vma, pt_op->entries); in unbind_op_prepare()
1741 xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, in unbind_op_prepare()
1742 pt_op->num_entries, false); in unbind_op_prepare()
1744 ++pt_update_ops->current_op; in unbind_op_prepare()
1745 pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma); in unbind_op_prepare()
1746 pt_update_ops->needs_invalidation = true; in unbind_op_prepare()
1748 xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries); in unbind_op_prepare()
1762 switch (op->base.op) { in op_prepare()
1764 if (!op->map.immediate && xe_vm_in_fault_mode(vm)) in op_prepare()
1767 err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma); in op_prepare()
1768 pt_update_ops->wait_vm_kernel = true; in op_prepare()
1772 gpuva_to_vma(op->base.remap.unmap->va)); in op_prepare()
1774 if (!err && op->remap.prev) { in op_prepare()
1776 op->remap.prev); in op_prepare()
1777 pt_update_ops->wait_vm_bookkeep = true; in op_prepare()
1779 if (!err && op->remap.next) { in op_prepare()
1781 op->remap.next); in op_prepare()
1782 pt_update_ops->wait_vm_bookkeep = true; in op_prepare()
1787 gpuva_to_vma(op->base.unmap.va)); in op_prepare()
1791 gpuva_to_vma(op->base.prefetch.va)); in op_prepare()
1792 pt_update_ops->wait_vm_kernel = true; in op_prepare()
1795 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_prepare()
1804 init_llist_head(&pt_update_ops->deferred); in xe_pt_update_ops_init()
1805 pt_update_ops->start = ~0x0ull; in xe_pt_update_ops_init()
1806 pt_update_ops->last = 0x0ull; in xe_pt_update_ops_init()
1810 * xe_pt_update_ops_prepare() - Prepare PT update operations
1823 &vops->pt_update_ops[tile->id]; in xe_pt_update_ops_prepare()
1825 int shift = tile->media_gt ? 1 : 0; in xe_pt_update_ops_prepare()
1828 lockdep_assert_held(&vops->vm->lock); in xe_pt_update_ops_prepare()
1829 xe_vm_assert_held(vops->vm); in xe_pt_update_ops_prepare()
1833 err = dma_resv_reserve_fences(xe_vm_resv(vops->vm), in xe_pt_update_ops_prepare()
1834 tile_to_xe(tile)->info.tile_count << shift); in xe_pt_update_ops_prepare()
1838 list_for_each_entry(op, &vops->list, link) { in xe_pt_update_ops_prepare()
1839 err = op_prepare(vops->vm, tile, pt_update_ops, op); in xe_pt_update_ops_prepare()
1845 xe_tile_assert(tile, pt_update_ops->current_op <= in xe_pt_update_ops_prepare()
1846 pt_update_ops->num_ops); in xe_pt_update_ops_prepare()
1849 if (vops->inject_error && in xe_pt_update_ops_prepare()
1850 vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE) in xe_pt_update_ops_prepare()
1851 return -ENOSPC; in xe_pt_update_ops_prepare()
1863 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { in bind_op_commit()
1864 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, in bind_op_commit()
1865 pt_update_ops->wait_vm_bookkeep ? in bind_op_commit()
1869 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, in bind_op_commit()
1870 pt_update_ops->wait_vm_bookkeep ? in bind_op_commit()
1874 vma->tile_present |= BIT(tile->id); in bind_op_commit()
1875 vma->tile_staged &= ~BIT(tile->id); in bind_op_commit()
1877 lockdep_assert_held_read(&vm->userptr.notifier_lock); in bind_op_commit()
1878 to_userptr_vma(vma)->userptr.initial_bind = true; in bind_op_commit()
1885 if (pt_update_ops->wait_vm_bookkeep && in bind_op_commit()
1887 !current->mm) in bind_op_commit()
1896 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { in unbind_op_commit()
1897 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, in unbind_op_commit()
1898 pt_update_ops->wait_vm_bookkeep ? in unbind_op_commit()
1902 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, in unbind_op_commit()
1903 pt_update_ops->wait_vm_bookkeep ? in unbind_op_commit()
1907 vma->tile_present &= ~BIT(tile->id); in unbind_op_commit()
1908 if (!vma->tile_present) { in unbind_op_commit()
1909 list_del_init(&vma->combined_links.rebind); in unbind_op_commit()
1911 lockdep_assert_held_read(&vm->userptr.notifier_lock); in unbind_op_commit()
1913 spin_lock(&vm->userptr.invalidated_lock); in unbind_op_commit()
1914 list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); in unbind_op_commit()
1915 spin_unlock(&vm->userptr.invalidated_lock); in unbind_op_commit()
1928 switch (op->base.op) { in op_commit()
1930 if (!op->map.immediate && xe_vm_in_fault_mode(vm)) in op_commit()
1933 bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence, in op_commit()
1938 gpuva_to_vma(op->base.remap.unmap->va), fence, in op_commit()
1941 if (op->remap.prev) in op_commit()
1942 bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, in op_commit()
1944 if (op->remap.next) in op_commit()
1945 bind_op_commit(vm, tile, pt_update_ops, op->remap.next, in op_commit()
1950 gpuva_to_vma(op->base.unmap.va), fence, fence2); in op_commit()
1954 gpuva_to_vma(op->base.prefetch.va), fence, fence2); in op_commit()
1957 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_commit()
1974 * xe_pt_update_ops_run() - Run PT update operations
1987 struct xe_vm *vm = vops->vm; in xe_pt_update_ops_run()
1989 &vops->pt_update_ops[tile->id]; in xe_pt_update_ops_run()
1998 .ops = pt_update_ops->needs_userptr_lock ? in xe_pt_update_ops_run()
2002 .tile_id = tile->id, in xe_pt_update_ops_run()
2005 lockdep_assert_held(&vm->lock); in xe_pt_update_ops_run()
2008 if (!pt_update_ops->current_op) { in xe_pt_update_ops_run()
2015 if (vops->inject_error && in xe_pt_update_ops_run()
2016 vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN) in xe_pt_update_ops_run()
2017 return ERR_PTR(-ENOSPC); in xe_pt_update_ops_run()
2020 if (pt_update_ops->needs_invalidation) { in xe_pt_update_ops_run()
2023 err = -ENOMEM; in xe_pt_update_ops_run()
2026 if (tile->media_gt) { in xe_pt_update_ops_run()
2029 err = -ENOMEM; in xe_pt_update_ops_run()
2034 err = -ENOMEM; in xe_pt_update_ops_run()
2039 err = -ENOMEM; in xe_pt_update_ops_run()
2047 err = -ENOMEM; in xe_pt_update_ops_run()
2051 fence = xe_migrate_update_pgtables(tile->migrate, &update); in xe_pt_update_ops_run()
2057 /* Point of no return - VM killed if failure after this */ in xe_pt_update_ops_run()
2058 for (i = 0; i < pt_update_ops->current_op; ++i) { in xe_pt_update_ops_run()
2059 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; in xe_pt_update_ops_run()
2061 xe_pt_commit(pt_op->vma, pt_op->entries, in xe_pt_update_ops_run()
2062 pt_op->num_entries, &pt_update_ops->deferred); in xe_pt_update_ops_run()
2063 pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */ in xe_pt_update_ops_run()
2066 if (xe_range_fence_insert(&vm->rftree[tile->id], rfence, in xe_pt_update_ops_run()
2068 pt_update_ops->start, in xe_pt_update_ops_run()
2069 pt_update_ops->last, fence)) in xe_pt_update_ops_run()
2076 invalidation_fence_init(tile->primary_gt, ifence, fence, in xe_pt_update_ops_run()
2077 pt_update_ops->start, in xe_pt_update_ops_run()
2078 pt_update_ops->last, vm->usm.asid); in xe_pt_update_ops_run()
2080 invalidation_fence_init(tile->media_gt, mfence, fence, in xe_pt_update_ops_run()
2081 pt_update_ops->start, in xe_pt_update_ops_run()
2082 pt_update_ops->last, vm->usm.asid); in xe_pt_update_ops_run()
2083 fences[0] = &ifence->base.base; in xe_pt_update_ops_run()
2084 fences[1] = &mfence->base.base; in xe_pt_update_ops_run()
2086 vm->composite_fence_ctx, in xe_pt_update_ops_run()
2087 vm->composite_fence_seqno++, in xe_pt_update_ops_run()
2089 fence = &cf->base; in xe_pt_update_ops_run()
2091 fence = &ifence->base.base; in xe_pt_update_ops_run()
2097 pt_update_ops->wait_vm_bookkeep ? in xe_pt_update_ops_run()
2101 list_for_each_entry(op, &vops->list, link) in xe_pt_update_ops_run()
2102 op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL); in xe_pt_update_ops_run()
2104 dma_resv_add_fence(xe_vm_resv(vm), &ifence->base.base, in xe_pt_update_ops_run()
2105 pt_update_ops->wait_vm_bookkeep ? in xe_pt_update_ops_run()
2109 dma_resv_add_fence(xe_vm_resv(vm), &mfence->base.base, in xe_pt_update_ops_run()
2110 pt_update_ops->wait_vm_bookkeep ? in xe_pt_update_ops_run()
2114 list_for_each_entry(op, &vops->list, link) in xe_pt_update_ops_run()
2115 op_commit(vops->vm, tile, pt_update_ops, op, in xe_pt_update_ops_run()
2116 &ifence->base.base, &mfence->base.base); in xe_pt_update_ops_run()
2119 if (pt_update_ops->needs_userptr_lock) in xe_pt_update_ops_run()
2120 up_read(&vm->userptr.notifier_lock); in xe_pt_update_ops_run()
2132 if (err != -EAGAIN && tile->id) in xe_pt_update_ops_run()
2133 xe_vm_kill(vops->vm, false); in xe_pt_update_ops_run()
2140 * xe_pt_update_ops_fini() - Finish PT update operations
2149 &vops->pt_update_ops[tile->id]; in xe_pt_update_ops_fini()
2152 lockdep_assert_held(&vops->vm->lock); in xe_pt_update_ops_fini()
2153 xe_vm_assert_held(vops->vm); in xe_pt_update_ops_fini()
2155 for (i = 0; i < pt_update_ops->current_op; ++i) { in xe_pt_update_ops_fini()
2156 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; in xe_pt_update_ops_fini()
2158 xe_pt_free_bind(pt_op->entries, pt_op->num_entries); in xe_pt_update_ops_fini()
2160 xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred); in xe_pt_update_ops_fini()
2164 * xe_pt_update_ops_abort() - Abort PT update operations
2173 &vops->pt_update_ops[tile->id]; in xe_pt_update_ops_abort()
2176 lockdep_assert_held(&vops->vm->lock); in xe_pt_update_ops_abort()
2177 xe_vm_assert_held(vops->vm); in xe_pt_update_ops_abort()
2179 for (i = pt_update_ops->num_ops - 1; i >= 0; --i) { in xe_pt_update_ops_abort()
2181 &pt_update_ops->ops[i]; in xe_pt_update_ops_abort()
2183 if (!pt_op->vma || i >= pt_update_ops->current_op) in xe_pt_update_ops_abort()
2186 if (pt_op->bind) in xe_pt_update_ops_abort()
2187 xe_pt_abort_bind(pt_op->vma, pt_op->entries, in xe_pt_update_ops_abort()
2188 pt_op->num_entries, in xe_pt_update_ops_abort()
2189 pt_op->rebind); in xe_pt_update_ops_abort()
2191 xe_pt_abort_unbind(pt_op->vma, pt_op->entries, in xe_pt_update_ops_abort()
2192 pt_op->num_entries); in xe_pt_update_ops_abort()