Lines Matching +full:tmr +full:- +full:manager
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
33 #include <linux/dma-mapping.h>
41 #include <linux/dma-buf.h>
78 return ttm_range_man_init(&adev->mman.bdev, type, in amdgpu_ttm_init_on_chip()
83 * amdgpu_evict_flags - Compute placement flags
93 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_evict_flags()
103 if (bo->type == ttm_bo_type_sg) { in amdgpu_evict_flags()
104 placement->num_placement = 0; in amdgpu_evict_flags()
110 placement->placement = &placements; in amdgpu_evict_flags()
111 placement->num_placement = 1; in amdgpu_evict_flags()
116 if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) { in amdgpu_evict_flags()
117 placement->num_placement = 0; in amdgpu_evict_flags()
121 switch (bo->resource->mem_type) { in amdgpu_evict_flags()
126 placement->num_placement = 0; in amdgpu_evict_flags()
130 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_evict_flags()
134 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && in amdgpu_evict_flags()
135 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && in amdgpu_evict_flags()
136 amdgpu_res_cpu_visible(adev, bo->resource)) { in amdgpu_evict_flags()
146 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; in amdgpu_evict_flags()
147 abo->placements[0].lpfn = 0; in amdgpu_evict_flags()
148 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; in amdgpu_evict_flags()
161 *placement = abo->placement; in amdgpu_evict_flags()
165 * amdgpu_ttm_map_buffer - Map memory into the GART windows
184 struct amdgpu_device *adev = ring->adev; in amdgpu_ttm_map_buffer()
193 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < in amdgpu_ttm_map_buffer()
196 if (WARN_ON(mem->mem_type == AMDGPU_PL_PREEMPT)) in amdgpu_ttm_map_buffer()
197 return -EINVAL; in amdgpu_ttm_map_buffer()
200 if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) { in amdgpu_ttm_map_buffer()
201 *addr = amdgpu_ttm_domain_start(adev, mem->mem_type) + in amdgpu_ttm_map_buffer()
202 mm_cur->start; in amdgpu_ttm_map_buffer()
211 offset = mm_cur->start & ~PAGE_MASK; in amdgpu_ttm_map_buffer()
216 *size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset); in amdgpu_ttm_map_buffer()
218 *addr = adev->gmc.gart_start; in amdgpu_ttm_map_buffer()
223 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_ttm_map_buffer()
226 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, in amdgpu_ttm_map_buffer()
234 src_addr += job->ibs[0].gpu_addr; in amdgpu_ttm_map_buffer()
236 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo); in amdgpu_ttm_map_buffer()
238 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, in amdgpu_ttm_map_buffer()
241 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_ttm_map_buffer()
242 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_map_buffer()
244 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem); in amdgpu_ttm_map_buffer()
248 cpu_addr = &job->ibs[0].ptr[num_dw]; in amdgpu_ttm_map_buffer()
250 if (mem->mem_type == TTM_PL_TT) { in amdgpu_ttm_map_buffer()
253 dma_addr = &bo->ttm->dma_address[mm_cur->start >> PAGE_SHIFT]; in amdgpu_ttm_map_buffer()
258 dma_address = mm_cur->start; in amdgpu_ttm_map_buffer()
259 dma_address += adev->vm_manager.vram_base_offset; in amdgpu_ttm_map_buffer()
273 * amdgpu_ttm_copy_mem_to_mem - Helper function for copy
282 * The function copies @size bytes from {src->mem + src->offset} to
283 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
294 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_copy_mem_to_mem()
301 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_ttm_copy_mem_to_mem()
303 return -EINVAL; in amdgpu_ttm_copy_mem_to_mem()
306 amdgpu_res_first(src->mem, src->offset, size, &src_mm); in amdgpu_ttm_copy_mem_to_mem()
307 amdgpu_res_first(dst->mem, dst->offset, size, &dst_mm); in amdgpu_ttm_copy_mem_to_mem()
309 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_ttm_copy_mem_to_mem()
319 r = amdgpu_ttm_map_buffer(src->bo, src->mem, &src_mm, in amdgpu_ttm_copy_mem_to_mem()
324 r = amdgpu_ttm_map_buffer(dst->bo, dst->mem, &dst_mm, in amdgpu_ttm_copy_mem_to_mem()
329 abo_src = ttm_to_amdgpu_bo(src->bo); in amdgpu_ttm_copy_mem_to_mem()
330 abo_dst = ttm_to_amdgpu_bo(dst->bo); in amdgpu_ttm_copy_mem_to_mem()
333 if ((abo_src->flags & AMDGPU_GEM_CREATE_GFX12_DCC) && in amdgpu_ttm_copy_mem_to_mem()
334 (abo_src->tbo.resource->mem_type == TTM_PL_VRAM)) in amdgpu_ttm_copy_mem_to_mem()
336 if ((abo_dst->flags & AMDGPU_GEM_CREATE_GFX12_DCC) && in amdgpu_ttm_copy_mem_to_mem()
337 (dst->mem->mem_type == TTM_PL_VRAM)) { in amdgpu_ttm_copy_mem_to_mem()
364 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_ttm_copy_mem_to_mem()
372 * amdgpu_move_blit - Copy an entire buffer to another buffer
382 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_move_blit()
396 new_mem->size, in amdgpu_move_blit()
398 bo->base.resv, &fence); in amdgpu_move_blit()
403 if (old_mem->mem_type == TTM_PL_VRAM && in amdgpu_move_blit()
404 (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) { in amdgpu_move_blit()
412 amdgpu_vram_mgr_set_cleared(bo->resource); in amdgpu_move_blit()
419 if (bo->type == ttm_bo_type_kernel) in amdgpu_move_blit()
434 * amdgpu_res_cpu_visible - Check that resource can be accessed by CPU
448 if (res->mem_type == TTM_PL_SYSTEM || res->mem_type == TTM_PL_TT || in amdgpu_res_cpu_visible()
449 res->mem_type == AMDGPU_PL_PREEMPT || res->mem_type == AMDGPU_PL_DOORBELL) in amdgpu_res_cpu_visible()
452 if (res->mem_type != TTM_PL_VRAM) in amdgpu_res_cpu_visible()
455 amdgpu_res_first(res, 0, res->size, &cursor); in amdgpu_res_cpu_visible()
457 if ((cursor.start + cursor.size) > adev->gmc.visible_vram_size) in amdgpu_res_cpu_visible()
466 * amdgpu_res_copyable - Check that memory can be accessed by ttm_bo_move_memcpy
477 if (mem->mem_type == TTM_PL_VRAM && in amdgpu_res_copyable()
478 !(mem->placement & TTM_PL_FLAG_CONTIGUOUS)) in amdgpu_res_copyable()
485 * amdgpu_bo_move - Move a buffer object to a new memory location
496 struct ttm_resource *old_mem = bo->resource; in amdgpu_bo_move()
499 if (new_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
500 new_mem->mem_type == AMDGPU_PL_PREEMPT) { in amdgpu_bo_move()
501 r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); in amdgpu_bo_move()
507 adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_bo_move()
509 if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
510 bo->ttm == NULL)) { in amdgpu_bo_move()
515 if (old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
516 (new_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
517 new_mem->mem_type == AMDGPU_PL_PREEMPT)) { in amdgpu_bo_move()
522 if ((old_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
523 old_mem->mem_type == AMDGPU_PL_PREEMPT) && in amdgpu_bo_move()
524 new_mem->mem_type == TTM_PL_SYSTEM) { in amdgpu_bo_move()
529 amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); in amdgpu_bo_move()
531 ttm_resource_free(bo, &bo->resource); in amdgpu_bo_move()
536 if (old_mem->mem_type == AMDGPU_PL_GDS || in amdgpu_bo_move()
537 old_mem->mem_type == AMDGPU_PL_GWS || in amdgpu_bo_move()
538 old_mem->mem_type == AMDGPU_PL_OA || in amdgpu_bo_move()
539 old_mem->mem_type == AMDGPU_PL_DOORBELL || in amdgpu_bo_move()
540 new_mem->mem_type == AMDGPU_PL_GDS || in amdgpu_bo_move()
541 new_mem->mem_type == AMDGPU_PL_GWS || in amdgpu_bo_move()
542 new_mem->mem_type == AMDGPU_PL_OA || in amdgpu_bo_move()
543 new_mem->mem_type == AMDGPU_PL_DOORBELL) { in amdgpu_bo_move()
550 if (bo->type == ttm_bo_type_device && in amdgpu_bo_move()
551 new_mem->mem_type == TTM_PL_VRAM && in amdgpu_bo_move()
552 old_mem->mem_type != TTM_PL_VRAM) { in amdgpu_bo_move()
553 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU in amdgpu_bo_move()
556 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; in amdgpu_bo_move()
559 if (adev->mman.buffer_funcs_enabled && in amdgpu_bo_move()
560 ((old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
561 new_mem->mem_type == TTM_PL_VRAM) || in amdgpu_bo_move()
562 (old_mem->mem_type == TTM_PL_VRAM && in amdgpu_bo_move()
563 new_mem->mem_type == TTM_PL_SYSTEM))) { in amdgpu_bo_move()
564 hop->fpfn = 0; in amdgpu_bo_move()
565 hop->lpfn = 0; in amdgpu_bo_move()
566 hop->mem_type = TTM_PL_TT; in amdgpu_bo_move()
567 hop->flags = TTM_PL_FLAG_TEMPORARY; in amdgpu_bo_move()
568 return -EMULTIHOP; in amdgpu_bo_move()
572 if (adev->mman.buffer_funcs_enabled) in amdgpu_bo_move()
575 r = -ENODEV; in amdgpu_bo_move()
592 atomic64_inc(&adev->num_evictions); in amdgpu_bo_move()
593 atomic64_add(bo->base.size, &adev->num_bytes_moved); in amdgpu_bo_move()
598 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
607 switch (mem->mem_type) { in amdgpu_ttm_io_mem_reserve()
615 mem->bus.offset = mem->start << PAGE_SHIFT; in amdgpu_ttm_io_mem_reserve()
617 if (adev->mman.aper_base_kaddr && in amdgpu_ttm_io_mem_reserve()
618 mem->placement & TTM_PL_FLAG_CONTIGUOUS) in amdgpu_ttm_io_mem_reserve()
619 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr + in amdgpu_ttm_io_mem_reserve()
620 mem->bus.offset; in amdgpu_ttm_io_mem_reserve()
622 mem->bus.offset += adev->gmc.aper_base; in amdgpu_ttm_io_mem_reserve()
623 mem->bus.is_iomem = true; in amdgpu_ttm_io_mem_reserve()
626 mem->bus.offset = mem->start << PAGE_SHIFT; in amdgpu_ttm_io_mem_reserve()
627 mem->bus.offset += adev->doorbell.base; in amdgpu_ttm_io_mem_reserve()
628 mem->bus.is_iomem = true; in amdgpu_ttm_io_mem_reserve()
629 mem->bus.caching = ttm_uncached; in amdgpu_ttm_io_mem_reserve()
632 return -EINVAL; in amdgpu_ttm_io_mem_reserve()
640 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_io_mem_pfn()
643 amdgpu_res_first(bo->resource, (u64)page_offset << PAGE_SHIFT, 0, in amdgpu_ttm_io_mem_pfn()
646 if (bo->resource->mem_type == AMDGPU_PL_DOORBELL) in amdgpu_ttm_io_mem_pfn()
647 return ((uint64_t)(adev->doorbell.base + cursor.start)) >> PAGE_SHIFT; in amdgpu_ttm_io_mem_pfn()
649 return (adev->gmc.aper_base + cursor.start) >> PAGE_SHIFT; in amdgpu_ttm_io_mem_pfn()
653 * amdgpu_ttm_domain_start - Returns GPU start address
665 return adev->gmc.gart_start; in amdgpu_ttm_domain_start()
667 return adev->gmc.vram_start; in amdgpu_ttm_domain_start()
691 * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
700 struct ttm_tt *ttm = bo->tbo.ttm; in amdgpu_ttm_tt_get_user_pages()
702 unsigned long start = gtt->userptr; in amdgpu_ttm_tt_get_user_pages()
711 mm = bo->notifier.mm; in amdgpu_ttm_tt_get_user_pages()
714 return -EFAULT; in amdgpu_ttm_tt_get_user_pages()
718 return -ESRCH; in amdgpu_ttm_tt_get_user_pages()
723 r = -EFAULT; in amdgpu_ttm_tt_get_user_pages()
726 if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) && in amdgpu_ttm_tt_get_user_pages()
727 vma->vm_file)) { in amdgpu_ttm_tt_get_user_pages()
728 r = -EPERM; in amdgpu_ttm_tt_get_user_pages()
733 r = amdgpu_hmm_range_get_pages(&bo->notifier, start, ttm->num_pages, in amdgpu_ttm_tt_get_user_pages()
745 /* amdgpu_ttm_tt_discard_user_pages - Discard range and pfn array allocations
752 if (gtt && gtt->userptr && range) in amdgpu_ttm_tt_discard_user_pages()
757 * amdgpu_ttm_tt_get_user_pages_done - stop HMM track the CPU page table change
767 if (!gtt || !gtt->userptr || !range) in amdgpu_ttm_tt_get_user_pages_done()
771 gtt->userptr, ttm->num_pages); in amdgpu_ttm_tt_get_user_pages_done()
773 WARN_ONCE(!range->hmm_pfns, "No user pages to check\n"); in amdgpu_ttm_tt_get_user_pages_done()
780 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
790 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_set_user_pages()
791 ttm->pages[i] = pages ? pages[i] : NULL; in amdgpu_ttm_tt_set_user_pages()
795 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
804 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_pin_userptr()
810 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, in amdgpu_ttm_tt_pin_userptr()
811 (u64)ttm->num_pages << PAGE_SHIFT, in amdgpu_ttm_tt_pin_userptr()
817 r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); in amdgpu_ttm_tt_pin_userptr()
822 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in amdgpu_ttm_tt_pin_userptr()
823 ttm->num_pages); in amdgpu_ttm_tt_pin_userptr()
828 sg_free_table(ttm->sg); in amdgpu_ttm_tt_pin_userptr()
830 kfree(ttm->sg); in amdgpu_ttm_tt_pin_userptr()
831 ttm->sg = NULL; in amdgpu_ttm_tt_pin_userptr()
836 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
843 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_unpin_userptr()
848 if (!ttm->sg || !ttm->sg->sgl) in amdgpu_ttm_tt_unpin_userptr()
852 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); in amdgpu_ttm_tt_unpin_userptr()
853 sg_free_table(ttm->sg); in amdgpu_ttm_tt_unpin_userptr()
867 uint64_t total_pages = ttm->num_pages; in amdgpu_ttm_gart_bind_gfx9_mqd()
868 int num_xcc = max(1U, adev->gfx.num_xcc_per_xcp); in amdgpu_ttm_gart_bind_gfx9_mqd()
879 gtt->offset + (page_idx << PAGE_SHIFT), in amdgpu_ttm_gart_bind_gfx9_mqd()
880 1, >t->ttm.dma_address[page_idx], flags); in amdgpu_ttm_gart_bind_gfx9_mqd()
882 * Ctrl pages - modify the memory type to NC (ctrl_flags) from in amdgpu_ttm_gart_bind_gfx9_mqd()
886 gtt->offset + ((page_idx + 1) << PAGE_SHIFT), in amdgpu_ttm_gart_bind_gfx9_mqd()
887 pages_per_xcc - 1, in amdgpu_ttm_gart_bind_gfx9_mqd()
888 >t->ttm.dma_address[page_idx + 1], in amdgpu_ttm_gart_bind_gfx9_mqd()
898 struct ttm_tt *ttm = tbo->ttm; in amdgpu_ttm_gart_bind()
904 if (abo->flags & AMDGPU_GEM_CREATE_CP_MQD_GFX9) { in amdgpu_ttm_gart_bind()
907 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, in amdgpu_ttm_gart_bind()
908 gtt->ttm.dma_address, flags); in amdgpu_ttm_gart_bind()
910 gtt->bound = true; in amdgpu_ttm_gart_bind()
914 * amdgpu_ttm_backend_bind - Bind GTT memory
929 return -EINVAL; in amdgpu_ttm_backend_bind()
931 if (gtt->bound) in amdgpu_ttm_backend_bind()
934 if (gtt->userptr) { in amdgpu_ttm_backend_bind()
940 } else if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) { in amdgpu_ttm_backend_bind()
941 if (!ttm->sg) { in amdgpu_ttm_backend_bind()
945 attach = gtt->gobj->import_attach; in amdgpu_ttm_backend_bind()
950 ttm->sg = sgt; in amdgpu_ttm_backend_bind()
953 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in amdgpu_ttm_backend_bind()
954 ttm->num_pages); in amdgpu_ttm_backend_bind()
957 if (!ttm->num_pages) { in amdgpu_ttm_backend_bind()
959 ttm->num_pages, bo_mem, ttm); in amdgpu_ttm_backend_bind()
962 if (bo_mem->mem_type != TTM_PL_TT || in amdgpu_ttm_backend_bind()
964 gtt->offset = AMDGPU_BO_INVALID_OFFSET; in amdgpu_ttm_backend_bind()
972 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT; in amdgpu_ttm_backend_bind()
973 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, in amdgpu_ttm_backend_bind()
974 gtt->ttm.dma_address, flags); in amdgpu_ttm_backend_bind()
975 gtt->bound = true; in amdgpu_ttm_backend_bind()
980 * amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either
989 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_alloc_gart()
991 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); in amdgpu_ttm_alloc_gart()
998 if (bo->resource->start != AMDGPU_BO_INVALID_OFFSET) in amdgpu_ttm_alloc_gart()
1009 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT; in amdgpu_ttm_alloc_gart()
1011 placements.flags = bo->resource->placement; in amdgpu_ttm_alloc_gart()
1018 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, tmp); in amdgpu_ttm_alloc_gart()
1021 gtt->offset = (u64)tmp->start << PAGE_SHIFT; in amdgpu_ttm_alloc_gart()
1024 ttm_resource_free(bo, &bo->resource); in amdgpu_ttm_alloc_gart()
1031 * amdgpu_ttm_recover_gart - Rebind GTT pages
1038 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); in amdgpu_ttm_recover_gart()
1041 if (!tbo->ttm) in amdgpu_ttm_recover_gart()
1044 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, tbo->resource); in amdgpu_ttm_recover_gart()
1049 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1061 if (gtt->userptr) { in amdgpu_ttm_backend_unbind()
1063 } else if (ttm->sg && gtt->gobj->import_attach) { in amdgpu_ttm_backend_unbind()
1066 attach = gtt->gobj->import_attach; in amdgpu_ttm_backend_unbind()
1067 dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL); in amdgpu_ttm_backend_unbind()
1068 ttm->sg = NULL; in amdgpu_ttm_backend_unbind()
1071 if (!gtt->bound) in amdgpu_ttm_backend_unbind()
1074 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET) in amdgpu_ttm_backend_unbind()
1078 amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages); in amdgpu_ttm_backend_unbind()
1079 gtt->bound = false; in amdgpu_ttm_backend_unbind()
1087 if (gtt->usertask) in amdgpu_ttm_backend_destroy()
1088 put_task_struct(gtt->usertask); in amdgpu_ttm_backend_destroy()
1090 ttm_tt_fini(>t->ttm); in amdgpu_ttm_backend_destroy()
1095 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1105 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_tt_create()
1114 gtt->gobj = &bo->base; in amdgpu_ttm_tt_create()
1115 if (adev->gmc.mem_partitions && abo->xcp_id >= 0) in amdgpu_ttm_tt_create()
1116 gtt->pool_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); in amdgpu_ttm_tt_create()
1118 gtt->pool_id = abo->xcp_id; in amdgpu_ttm_tt_create()
1120 if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) in amdgpu_ttm_tt_create()
1126 if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) { in amdgpu_ttm_tt_create()
1130 return >t->ttm; in amdgpu_ttm_tt_create()
1134 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1150 if (gtt->userptr) { in amdgpu_ttm_tt_populate()
1151 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); in amdgpu_ttm_tt_populate()
1152 if (!ttm->sg) in amdgpu_ttm_tt_populate()
1153 return -ENOMEM; in amdgpu_ttm_tt_populate()
1157 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) in amdgpu_ttm_tt_populate()
1160 if (adev->mman.ttm_pools && gtt->pool_id >= 0) in amdgpu_ttm_tt_populate()
1161 pool = &adev->mman.ttm_pools[gtt->pool_id]; in amdgpu_ttm_tt_populate()
1163 pool = &adev->mman.bdev.pool; in amdgpu_ttm_tt_populate()
1168 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_populate()
1169 ttm->pages[i]->mapping = bdev->dev_mapping; in amdgpu_ttm_tt_populate()
1175 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1190 if (gtt->userptr) { in amdgpu_ttm_tt_unpopulate()
1192 kfree(ttm->sg); in amdgpu_ttm_tt_unpopulate()
1193 ttm->sg = NULL; in amdgpu_ttm_tt_unpopulate()
1197 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) in amdgpu_ttm_tt_unpopulate()
1200 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_unpopulate()
1201 ttm->pages[i]->mapping = NULL; in amdgpu_ttm_tt_unpopulate()
1205 if (adev->mman.ttm_pools && gtt->pool_id >= 0) in amdgpu_ttm_tt_unpopulate()
1206 pool = &adev->mman.ttm_pools[gtt->pool_id]; in amdgpu_ttm_tt_unpopulate()
1208 pool = &adev->mman.bdev.pool; in amdgpu_ttm_tt_unpopulate()
1214 * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current
1225 if (!tbo->ttm) in amdgpu_ttm_tt_get_userptr()
1226 return -EINVAL; in amdgpu_ttm_tt_get_userptr()
1228 gtt = (void *)tbo->ttm; in amdgpu_ttm_tt_get_userptr()
1229 *user_addr = gtt->userptr; in amdgpu_ttm_tt_get_userptr()
1234 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1250 if (!bo->ttm) { in amdgpu_ttm_tt_set_userptr()
1252 bo->ttm = amdgpu_ttm_tt_create(bo, 0); in amdgpu_ttm_tt_set_userptr()
1253 if (bo->ttm == NULL) in amdgpu_ttm_tt_set_userptr()
1254 return -ENOMEM; in amdgpu_ttm_tt_set_userptr()
1258 bo->ttm->page_flags |= TTM_TT_FLAG_EXTERNAL; in amdgpu_ttm_tt_set_userptr()
1260 gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); in amdgpu_ttm_tt_set_userptr()
1261 gtt->userptr = addr; in amdgpu_ttm_tt_set_userptr()
1262 gtt->userflags = flags; in amdgpu_ttm_tt_set_userptr()
1264 if (gtt->usertask) in amdgpu_ttm_tt_set_userptr()
1265 put_task_struct(gtt->usertask); in amdgpu_ttm_tt_set_userptr()
1266 gtt->usertask = current->group_leader; in amdgpu_ttm_tt_set_userptr()
1267 get_task_struct(gtt->usertask); in amdgpu_ttm_tt_set_userptr()
1273 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1282 if (gtt->usertask == NULL) in amdgpu_ttm_tt_get_usermm()
1285 return gtt->usertask->mm; in amdgpu_ttm_tt_get_usermm()
1289 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1299 if (gtt == NULL || !gtt->userptr) in amdgpu_ttm_tt_affect_userptr()
1305 size = (unsigned long)gtt->ttm.num_pages * PAGE_SIZE; in amdgpu_ttm_tt_affect_userptr()
1306 if (gtt->userptr > end || gtt->userptr + size <= start) in amdgpu_ttm_tt_affect_userptr()
1310 *userptr = gtt->userptr; in amdgpu_ttm_tt_affect_userptr()
1315 * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1321 if (gtt == NULL || !gtt->userptr) in amdgpu_ttm_tt_is_userptr()
1328 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1337 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_is_readonly()
1341 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1352 if (mem && mem->mem_type != TTM_PL_SYSTEM) in amdgpu_ttm_tt_pde_flags()
1355 if (mem && (mem->mem_type == TTM_PL_TT || in amdgpu_ttm_tt_pde_flags()
1356 mem->mem_type == AMDGPU_PL_DOORBELL || in amdgpu_ttm_tt_pde_flags()
1357 mem->mem_type == AMDGPU_PL_PREEMPT)) { in amdgpu_ttm_tt_pde_flags()
1360 if (ttm->caching == ttm_cached) in amdgpu_ttm_tt_pde_flags()
1364 if (mem && mem->mem_type == TTM_PL_VRAM && in amdgpu_ttm_tt_pde_flags()
1365 mem->bus.caching == ttm_cached) in amdgpu_ttm_tt_pde_flags()
1372 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1385 flags |= adev->gart.gart_pte_flags; in amdgpu_ttm_tt_pte_flags()
1395 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1413 if (bo->resource->mem_type == TTM_PL_SYSTEM) in amdgpu_ttm_bo_eviction_valuable()
1416 if (bo->type == ttm_bo_type_kernel && in amdgpu_ttm_bo_eviction_valuable()
1424 dma_resv_for_each_fence(&resv_cursor, bo->base.resv, in amdgpu_ttm_bo_eviction_valuable()
1426 if (amdkfd_fence_check_mm(f, current->mm) && in amdgpu_ttm_bo_eviction_valuable()
1427 !(place->flags & TTM_PL_FLAG_CONTIGUOUS)) in amdgpu_ttm_bo_eviction_valuable()
1438 if (bo->resource->mem_type == AMDGPU_PL_PREEMPT) in amdgpu_ttm_bo_eviction_valuable()
1441 if (bo->resource->mem_type == TTM_PL_TT && in amdgpu_ttm_bo_eviction_valuable()
1453 uint64_t bytes = 4 - (pos & 0x3); in amdgpu_ttm_vram_mm_access()
1459 mask &= 0xffffffff >> (bytes - size) * 8; in amdgpu_ttm_vram_mm_access()
1479 size -= bytes; in amdgpu_ttm_vram_mm_access()
1488 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); in amdgpu_ttm_access_memory_sdma()
1497 return -EINVAL; in amdgpu_ttm_access_memory_sdma()
1499 if (!adev->mman.sdma_access_ptr) in amdgpu_ttm_access_memory_sdma()
1500 return -EACCES; in amdgpu_ttm_access_memory_sdma()
1503 return -ENODEV; in amdgpu_ttm_access_memory_sdma()
1506 memcpy(adev->mman.sdma_access_ptr, buf, len); in amdgpu_ttm_access_memory_sdma()
1508 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_ttm_access_memory_sdma()
1509 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, in amdgpu_ttm_access_memory_sdma()
1516 amdgpu_res_first(abo->tbo.resource, offset, len, &src_mm); in amdgpu_ttm_access_memory_sdma()
1517 src_addr = amdgpu_ttm_domain_start(adev, bo->resource->mem_type) + in amdgpu_ttm_access_memory_sdma()
1519 dst_addr = amdgpu_bo_gpu_offset(adev->mman.sdma_access_bo); in amdgpu_ttm_access_memory_sdma()
1523 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr, in amdgpu_ttm_access_memory_sdma()
1526 amdgpu_ring_pad_ib(adev->mman.buffer_funcs_ring, &job->ibs[0]); in amdgpu_ttm_access_memory_sdma()
1527 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_access_memory_sdma()
1531 if (!dma_fence_wait_timeout(fence, false, adev->sdma_timeout)) in amdgpu_ttm_access_memory_sdma()
1532 r = -ETIMEDOUT; in amdgpu_ttm_access_memory_sdma()
1536 memcpy(buf, adev->mman.sdma_access_ptr, len); in amdgpu_ttm_access_memory_sdma()
1543 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1559 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); in amdgpu_ttm_access_memory()
1563 if (bo->resource->mem_type != TTM_PL_VRAM) in amdgpu_ttm_access_memory()
1564 return -EIO; in amdgpu_ttm_access_memory()
1570 amdgpu_res_first(bo->resource, offset, len, &cursor); in amdgpu_ttm_access_memory()
1576 size -= count; in amdgpu_ttm_access_memory()
1578 /* using MM to access rest vram and handle un-aligned address */ in amdgpu_ttm_access_memory()
1617 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1625 amdgpu_bo_free_kernel(&adev->mman.fw_vram_usage_reserved_bo, in amdgpu_ttm_fw_reserve_vram_fini()
1626 NULL, &adev->mman.fw_vram_usage_va); in amdgpu_ttm_fw_reserve_vram_fini()
1633 * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
1641 amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo, in amdgpu_ttm_drv_reserve_vram_fini()
1643 &adev->mman.drv_vram_usage_va); in amdgpu_ttm_drv_reserve_vram_fini()
1647 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1655 uint64_t vram_size = adev->gmc.visible_vram_size; in amdgpu_ttm_fw_reserve_vram_init()
1657 adev->mman.fw_vram_usage_va = NULL; in amdgpu_ttm_fw_reserve_vram_init()
1658 adev->mman.fw_vram_usage_reserved_bo = NULL; in amdgpu_ttm_fw_reserve_vram_init()
1660 if (adev->mman.fw_vram_usage_size == 0 || in amdgpu_ttm_fw_reserve_vram_init()
1661 adev->mman.fw_vram_usage_size > vram_size) in amdgpu_ttm_fw_reserve_vram_init()
1665 adev->mman.fw_vram_usage_start_offset, in amdgpu_ttm_fw_reserve_vram_init()
1666 adev->mman.fw_vram_usage_size, in amdgpu_ttm_fw_reserve_vram_init()
1667 &adev->mman.fw_vram_usage_reserved_bo, in amdgpu_ttm_fw_reserve_vram_init()
1668 &adev->mman.fw_vram_usage_va); in amdgpu_ttm_fw_reserve_vram_init()
1672 * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
1680 u64 vram_size = adev->gmc.visible_vram_size; in amdgpu_ttm_drv_reserve_vram_init()
1682 adev->mman.drv_vram_usage_va = NULL; in amdgpu_ttm_drv_reserve_vram_init()
1683 adev->mman.drv_vram_usage_reserved_bo = NULL; in amdgpu_ttm_drv_reserve_vram_init()
1685 if (adev->mman.drv_vram_usage_size == 0 || in amdgpu_ttm_drv_reserve_vram_init()
1686 adev->mman.drv_vram_usage_size > vram_size) in amdgpu_ttm_drv_reserve_vram_init()
1690 adev->mman.drv_vram_usage_start_offset, in amdgpu_ttm_drv_reserve_vram_init()
1691 adev->mman.drv_vram_usage_size, in amdgpu_ttm_drv_reserve_vram_init()
1692 &adev->mman.drv_vram_usage_reserved_bo, in amdgpu_ttm_drv_reserve_vram_init()
1693 &adev->mman.drv_vram_usage_va); in amdgpu_ttm_drv_reserve_vram_init()
1701 * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
1709 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_training_reserve_vram_fini()
1711 ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT; in amdgpu_ttm_training_reserve_vram_fini()
1712 amdgpu_bo_free_kernel(&ctx->c2p_bo, NULL, NULL); in amdgpu_ttm_training_reserve_vram_fini()
1713 ctx->c2p_bo = NULL; in amdgpu_ttm_training_reserve_vram_fini()
1721 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_training_data_block_init()
1725 ctx->c2p_train_data_offset = in amdgpu_ttm_training_data_block_init()
1726 ALIGN((adev->gmc.mc_vram_size - reserve_size - SZ_1M), SZ_1M); in amdgpu_ttm_training_data_block_init()
1727 ctx->p2c_train_data_offset = in amdgpu_ttm_training_data_block_init()
1728 (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET); in amdgpu_ttm_training_data_block_init()
1729 ctx->train_data_size = in amdgpu_ttm_training_data_block_init()
1733 ctx->train_data_size, in amdgpu_ttm_training_data_block_init()
1734 ctx->p2c_train_data_offset, in amdgpu_ttm_training_data_block_init()
1735 ctx->c2p_train_data_offset); in amdgpu_ttm_training_data_block_init()
1739 * reserve TMR memory at the top of VRAM which holds
1744 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_reserve_tmr()
1749 if (adev->bios && !amdgpu_sriov_vf(adev)) { in amdgpu_ttm_reserve_tmr()
1757 * Query reserved tmr size through atom firmwareinfo for Sienna_Cichlid and onwards for all in amdgpu_ttm_reserve_tmr()
1760 * Otherwise, fallback to legacy approach to check and reserve tmr block for ip in amdgpu_ttm_reserve_tmr()
1763 if (adev->bios) in amdgpu_ttm_reserve_tmr()
1767 if (!adev->bios && in amdgpu_ttm_reserve_tmr()
1776 /* reserve vram for mem train according to TMR location */ in amdgpu_ttm_reserve_tmr()
1779 ctx->c2p_train_data_offset, in amdgpu_ttm_reserve_tmr()
1780 ctx->train_data_size, in amdgpu_ttm_reserve_tmr()
1781 &ctx->c2p_bo, in amdgpu_ttm_reserve_tmr()
1788 ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS; in amdgpu_ttm_reserve_tmr()
1791 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_reserve_tmr()
1793 adev, adev->gmc.real_vram_size - reserve_size, in amdgpu_ttm_reserve_tmr()
1794 reserve_size, &adev->mman.fw_reserved_memory, NULL); in amdgpu_ttm_reserve_tmr()
1796 DRM_ERROR("alloc tmr failed(%d)!\n", ret); in amdgpu_ttm_reserve_tmr()
1797 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, in amdgpu_ttm_reserve_tmr()
1802 DRM_DEBUG_DRIVER("backdoor fw loading path for PSP TMR, no reservation needed\n"); in amdgpu_ttm_reserve_tmr()
1812 if (!adev->gmc.is_app_apu || !adev->gmc.num_mem_partitions) in amdgpu_ttm_pools_init()
1815 adev->mman.ttm_pools = kcalloc(adev->gmc.num_mem_partitions, in amdgpu_ttm_pools_init()
1816 sizeof(*adev->mman.ttm_pools), in amdgpu_ttm_pools_init()
1818 if (!adev->mman.ttm_pools) in amdgpu_ttm_pools_init()
1819 return -ENOMEM; in amdgpu_ttm_pools_init()
1821 for (i = 0; i < adev->gmc.num_mem_partitions; i++) { in amdgpu_ttm_pools_init()
1822 ttm_pool_init(&adev->mman.ttm_pools[i], adev->dev, in amdgpu_ttm_pools_init()
1823 adev->gmc.mem_partitions[i].numa.node, in amdgpu_ttm_pools_init()
1833 if (!adev->gmc.is_app_apu || !adev->mman.ttm_pools) in amdgpu_ttm_pools_fini()
1836 for (i = 0; i < adev->gmc.num_mem_partitions; i++) in amdgpu_ttm_pools_fini()
1837 ttm_pool_fini(&adev->mman.ttm_pools[i]); in amdgpu_ttm_pools_fini()
1839 kfree(adev->mman.ttm_pools); in amdgpu_ttm_pools_fini()
1840 adev->mman.ttm_pools = NULL; in amdgpu_ttm_pools_fini()
1844 * amdgpu_ttm_init - Init the memory management (ttm) as well as various
1849 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1857 mutex_init(&adev->mman.gtt_window_lock); in amdgpu_ttm_init()
1859 dma_set_max_seg_size(adev->dev, UINT_MAX); in amdgpu_ttm_init()
1861 r = ttm_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev, in amdgpu_ttm_init()
1862 adev_to_drm(adev)->anon_inode->i_mapping, in amdgpu_ttm_init()
1863 adev_to_drm(adev)->vma_offset_manager, in amdgpu_ttm_init()
1864 adev->need_swiotlb, in amdgpu_ttm_init()
1865 dma_addressing_limited(adev->dev)); in amdgpu_ttm_init()
1876 adev->mman.initialized = true; in amdgpu_ttm_init()
1889 if (adev->gmc.xgmi.connected_to_cpu) in amdgpu_ttm_init()
1890 adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base, in amdgpu_ttm_init()
1891 adev->gmc.visible_vram_size); in amdgpu_ttm_init()
1893 else if (adev->gmc.is_app_apu) in amdgpu_ttm_init()
1898 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base, in amdgpu_ttm_init()
1899 adev->gmc.visible_vram_size); in amdgpu_ttm_init()
1923 if (adev->mman.discovery_bin) { in amdgpu_ttm_init()
1930 * This is used for VGA emulation and pre-OS scanout buffers to in amdgpu_ttm_init()
1931 * avoid display artifacts while transitioning between pre-OS in amdgpu_ttm_init()
1934 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_init()
1936 adev->mman.stolen_vga_size, in amdgpu_ttm_init()
1937 &adev->mman.stolen_vga_memory, in amdgpu_ttm_init()
1942 r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size, in amdgpu_ttm_init()
1943 adev->mman.stolen_extended_size, in amdgpu_ttm_init()
1944 &adev->mman.stolen_extended_memory, in amdgpu_ttm_init()
1951 adev->mman.stolen_reserved_offset, in amdgpu_ttm_init()
1952 adev->mman.stolen_reserved_size, in amdgpu_ttm_init()
1953 &adev->mman.stolen_reserved_memory, in amdgpu_ttm_init()
1962 (unsigned int)(adev->gmc.real_vram_size / (1024 * 1024))); in amdgpu_ttm_init()
1967 if (amdgpu_gtt_size == -1) in amdgpu_ttm_init()
1982 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_DOORBELL, adev->doorbell.size / PAGE_SIZE); in amdgpu_ttm_init()
2002 /* Initialize various on-chip memory pools */ in amdgpu_ttm_init()
2003 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, adev->gds.gds_size); in amdgpu_ttm_init()
2009 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, adev->gds.gws_size); in amdgpu_ttm_init()
2015 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, adev->gds.oa_size); in amdgpu_ttm_init()
2022 &adev->mman.sdma_access_bo, NULL, in amdgpu_ttm_init()
2023 &adev->mman.sdma_access_ptr)) in amdgpu_ttm_init()
2030 * amdgpu_ttm_fini - De-initialize the TTM memory pools
2036 if (!adev->mman.initialized) in amdgpu_ttm_fini()
2043 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_fini()
2044 amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); in amdgpu_ttm_fini()
2045 amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL); in amdgpu_ttm_fini()
2047 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, NULL, in amdgpu_ttm_fini()
2049 if (adev->mman.stolen_reserved_size) in amdgpu_ttm_fini()
2050 amdgpu_bo_free_kernel(&adev->mman.stolen_reserved_memory, in amdgpu_ttm_fini()
2053 amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL, in amdgpu_ttm_fini()
2054 &adev->mman.sdma_access_ptr); in amdgpu_ttm_fini()
2060 if (adev->mman.aper_base_kaddr) in amdgpu_ttm_fini()
2061 iounmap(adev->mman.aper_base_kaddr); in amdgpu_ttm_fini()
2062 adev->mman.aper_base_kaddr = NULL; in amdgpu_ttm_fini()
2070 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); in amdgpu_ttm_fini()
2071 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS); in amdgpu_ttm_fini()
2072 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA); in amdgpu_ttm_fini()
2073 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_DOORBELL); in amdgpu_ttm_fini()
2074 ttm_device_fini(&adev->mman.bdev); in amdgpu_ttm_fini()
2075 adev->mman.initialized = false; in amdgpu_ttm_fini()
2080 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
2090 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); in amdgpu_ttm_set_buffer_funcs_status()
2094 if (!adev->mman.initialized || amdgpu_in_reset(adev) || in amdgpu_ttm_set_buffer_funcs_status()
2095 adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu) in amdgpu_ttm_set_buffer_funcs_status()
2102 ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_set_buffer_funcs_status()
2103 sched = &ring->sched; in amdgpu_ttm_set_buffer_funcs_status()
2104 r = drm_sched_entity_init(&adev->mman.high_pr, in amdgpu_ttm_set_buffer_funcs_status()
2113 r = drm_sched_entity_init(&adev->mman.low_pr, in amdgpu_ttm_set_buffer_funcs_status()
2122 drm_sched_entity_destroy(&adev->mman.high_pr); in amdgpu_ttm_set_buffer_funcs_status()
2123 drm_sched_entity_destroy(&adev->mman.low_pr); in amdgpu_ttm_set_buffer_funcs_status()
2124 dma_fence_put(man->move); in amdgpu_ttm_set_buffer_funcs_status()
2125 man->move = NULL; in amdgpu_ttm_set_buffer_funcs_status()
2130 size = adev->gmc.real_vram_size; in amdgpu_ttm_set_buffer_funcs_status()
2132 size = adev->gmc.visible_vram_size; in amdgpu_ttm_set_buffer_funcs_status()
2133 man->size = size; in amdgpu_ttm_set_buffer_funcs_status()
2134 adev->mman.buffer_funcs_enabled = enable; in amdgpu_ttm_set_buffer_funcs_status()
2139 drm_sched_entity_destroy(&adev->mman.high_pr); in amdgpu_ttm_set_buffer_funcs_status()
2154 struct drm_sched_entity *entity = delayed ? &adev->mman.low_pr : in amdgpu_ttm_prepare_job()
2155 &adev->mman.high_pr; in amdgpu_ttm_prepare_job()
2163 (*job)->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo ? in amdgpu_ttm_prepare_job()
2164 adev->gmc.pdb0_bo : in amdgpu_ttm_prepare_job()
2165 adev->gart.bo); in amdgpu_ttm_prepare_job()
2166 (*job)->vm_needs_flush = true; in amdgpu_ttm_prepare_job()
2171 return drm_sched_job_add_resv_dependencies(&(*job)->base, resv, in amdgpu_ttm_prepare_job()
2181 struct amdgpu_device *adev = ring->adev; in amdgpu_copy_buffer()
2188 if (!direct_submit && !ring->sched.ready) { in amdgpu_copy_buffer()
2190 return -EINVAL; in amdgpu_copy_buffer()
2193 max_bytes = adev->mman.buffer_funcs->copy_max_bytes; in amdgpu_copy_buffer()
2195 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_copy_buffer()
2204 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset, in amdgpu_copy_buffer()
2208 byte_count -= cur_size_in_bytes; in amdgpu_copy_buffer()
2211 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_copy_buffer()
2212 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_copy_buffer()
2234 struct amdgpu_device *adev = ring->adev; in amdgpu_ttm_fill_mem()
2241 max_bytes = adev->mman.buffer_funcs->fill_max_bytes; in amdgpu_ttm_fill_mem()
2243 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8); in amdgpu_ttm_fill_mem()
2252 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, dst_addr, in amdgpu_ttm_fill_mem()
2256 byte_count -= cur_size; in amdgpu_ttm_fill_mem()
2259 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_ttm_fill_mem()
2260 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_fill_mem()
2266 * amdgpu_ttm_clear_buffer - clear memory buffers
2280 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_ttm_clear_buffer()
2281 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_clear_buffer()
2286 if (!adev->mman.buffer_funcs_enabled) in amdgpu_ttm_clear_buffer()
2287 return -EINVAL; in amdgpu_ttm_clear_buffer()
2290 return -EINVAL; in amdgpu_ttm_clear_buffer()
2294 amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor); in amdgpu_ttm_clear_buffer()
2296 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_ttm_clear_buffer()
2309 r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &cursor, in amdgpu_ttm_clear_buffer()
2325 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_ttm_clear_buffer()
2336 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_fill_buffer()
2337 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_fill_buffer()
2342 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_fill_buffer()
2344 return -EINVAL; in amdgpu_fill_buffer()
2347 amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &dst); in amdgpu_fill_buffer()
2349 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_fill_buffer()
2357 r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &dst, in amdgpu_fill_buffer()
2373 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_fill_buffer()
2381 * amdgpu_ttm_evict_resources - evict memory buffers
2400 man = ttm_manager_type(&adev->mman.bdev, mem_type); in amdgpu_ttm_evict_resources()
2404 return -EINVAL; in amdgpu_ttm_evict_resources()
2407 return ttm_resource_manager_evict_all(&adev->mman.bdev, man); in amdgpu_ttm_evict_resources()
2414 struct amdgpu_device *adev = m->private; in amdgpu_ttm_page_pool_show()
2416 return ttm_pool_debugfs(&adev->mman.bdev.pool, m); in amdgpu_ttm_page_pool_show()
2422 * amdgpu_ttm_vram_read - Linear read access to VRAM
2429 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_ttm_vram_read()
2433 return -EINVAL; in amdgpu_ttm_vram_read()
2435 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_read()
2436 return -ENXIO; in amdgpu_ttm_vram_read()
2438 size = min(size, (size_t)(adev->gmc.mc_vram_size - *pos)); in amdgpu_ttm_vram_read()
2445 return -EFAULT; in amdgpu_ttm_vram_read()
2450 size -= bytes; in amdgpu_ttm_vram_read()
2457 * amdgpu_ttm_vram_write - Linear write access to VRAM
2464 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_ttm_vram_write()
2469 return -EINVAL; in amdgpu_ttm_vram_write()
2471 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_write()
2472 return -ENXIO; in amdgpu_ttm_vram_write()
2477 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_write()
2489 size -= 4; in amdgpu_ttm_vram_write()
2503 * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2512 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_iomem_read()
2518 dom = iommu_get_domain_for_dev(adev->dev); in amdgpu_iomem_read()
2523 size_t bytes = PAGE_SIZE - off; in amdgpu_iomem_read()
2538 return -EPERM; in amdgpu_iomem_read()
2541 if (p->mapping != adev->mman.bdev.dev_mapping) in amdgpu_iomem_read()
2542 return -EPERM; in amdgpu_iomem_read()
2548 return -EFAULT; in amdgpu_iomem_read()
2550 size -= bytes; in amdgpu_iomem_read()
2559 * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2568 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_iomem_write()
2573 dom = iommu_get_domain_for_dev(adev->dev); in amdgpu_iomem_write()
2578 size_t bytes = PAGE_SIZE - off; in amdgpu_iomem_write()
2589 return -EPERM; in amdgpu_iomem_write()
2592 if (p->mapping != adev->mman.bdev.dev_mapping) in amdgpu_iomem_write()
2593 return -EPERM; in amdgpu_iomem_write()
2599 return -EFAULT; in amdgpu_iomem_write()
2601 size -= bytes; in amdgpu_iomem_write()
2621 struct drm_minor *minor = adev_to_drm(adev)->primary; in amdgpu_ttm_debugfs_init()
2622 struct dentry *root = minor->debugfs_root; in amdgpu_ttm_debugfs_init()
2625 &amdgpu_ttm_vram_fops, adev->gmc.mc_vram_size); in amdgpu_ttm_debugfs_init()
2630 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2633 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2636 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2639 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2642 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()