Lines Matching +full:mc +full:- +full:bus
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
34 #include <linux/dma-mapping.h>
72 return ttm_range_man_init(&rdev->mman.bdev, TTM_PL_VRAM, in radeon_ttm_init_vram()
73 false, rdev->mc.real_vram_size >> PAGE_SHIFT); in radeon_ttm_init_vram()
78 return ttm_range_man_init(&rdev->mman.bdev, TTM_PL_TT, in radeon_ttm_init_gtt()
79 true, rdev->mc.gtt_size >> PAGE_SHIFT); in radeon_ttm_init_gtt()
95 placement->placement = &placements; in radeon_evict_flags()
96 placement->num_placement = 1; in radeon_evict_flags()
100 switch (bo->resource->mem_type) { in radeon_evict_flags()
102 if (rbo->rdev->ring[radeon_copy_ring_index(rbo->rdev)].ready == false) in radeon_evict_flags()
104 else if (rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size && in radeon_evict_flags()
105 bo->resource->start < (rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT)) { in radeon_evict_flags()
106 unsigned fpfn = rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; in radeon_evict_flags()
116 for (i = 0; i < rbo->placement.num_placement; i++) { in radeon_evict_flags()
117 if (rbo->placements[i].mem_type == TTM_PL_VRAM) { in radeon_evict_flags()
118 if (rbo->placements[i].fpfn < fpfn) in radeon_evict_flags()
119 rbo->placements[i].fpfn = fpfn; in radeon_evict_flags()
120 rbo->placements[0].flags |= TTM_PL_FLAG_DESIRED; in radeon_evict_flags()
130 *placement = rbo->placement; in radeon_evict_flags()
144 rdev = radeon_get_rdev(bo->bdev); in radeon_move_blit()
146 old_start = (u64)old_mem->start << PAGE_SHIFT; in radeon_move_blit()
147 new_start = (u64)new_mem->start << PAGE_SHIFT; in radeon_move_blit()
149 switch (old_mem->mem_type) { in radeon_move_blit()
151 old_start += rdev->mc.vram_start; in radeon_move_blit()
154 old_start += rdev->mc.gtt_start; in radeon_move_blit()
157 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); in radeon_move_blit()
158 return -EINVAL; in radeon_move_blit()
160 switch (new_mem->mem_type) { in radeon_move_blit()
162 new_start += rdev->mc.vram_start; in radeon_move_blit()
165 new_start += rdev->mc.gtt_start; in radeon_move_blit()
168 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); in radeon_move_blit()
169 return -EINVAL; in radeon_move_blit()
171 if (!rdev->ring[ridx].ready) { in radeon_move_blit()
173 return -EINVAL; in radeon_move_blit()
178 num_pages = PFN_UP(new_mem->size) * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); in radeon_move_blit()
179 fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->base.resv); in radeon_move_blit()
183 r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, false, new_mem); in radeon_move_blit()
193 struct ttm_resource *old_mem = bo->resource; in radeon_bo_move()
197 if (new_mem->mem_type == TTM_PL_TT) { in radeon_bo_move()
198 r = radeon_ttm_tt_bind(bo->bdev, bo->ttm, new_mem); in radeon_bo_move()
207 rdev = radeon_get_rdev(bo->bdev); in radeon_bo_move()
208 if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM && in radeon_bo_move()
209 bo->ttm == NULL)) { in radeon_bo_move()
213 if (old_mem->mem_type == TTM_PL_SYSTEM && in radeon_bo_move()
214 new_mem->mem_type == TTM_PL_TT) { in radeon_bo_move()
219 if (old_mem->mem_type == TTM_PL_TT && in radeon_bo_move()
220 new_mem->mem_type == TTM_PL_SYSTEM) { in radeon_bo_move()
221 radeon_ttm_tt_unbind(bo->bdev, bo->ttm); in radeon_bo_move()
225 if (rdev->ring[radeon_copy_ring_index(rdev)].ready && in radeon_bo_move()
226 rdev->asic->copy.copy != NULL) { in radeon_bo_move()
227 if ((old_mem->mem_type == TTM_PL_SYSTEM && in radeon_bo_move()
228 new_mem->mem_type == TTM_PL_VRAM) || in radeon_bo_move()
229 (old_mem->mem_type == TTM_PL_VRAM && in radeon_bo_move()
230 new_mem->mem_type == TTM_PL_SYSTEM)) { in radeon_bo_move()
231 hop->fpfn = 0; in radeon_bo_move()
232 hop->lpfn = 0; in radeon_bo_move()
233 hop->mem_type = TTM_PL_TT; in radeon_bo_move()
234 hop->flags = 0; in radeon_bo_move()
235 return -EMULTIHOP; in radeon_bo_move()
240 r = -ENODEV; in radeon_bo_move()
251 atomic64_add(bo->base.size, &rdev->num_bytes_moved); in radeon_bo_move()
259 size_t bus_size = (size_t)mem->size; in radeon_ttm_io_mem_reserve()
261 switch (mem->mem_type) { in radeon_ttm_io_mem_reserve()
267 if (rdev->flags & RADEON_IS_AGP) { in radeon_ttm_io_mem_reserve()
269 mem->bus.offset = (mem->start << PAGE_SHIFT) + in radeon_ttm_io_mem_reserve()
270 rdev->mc.agp_base; in radeon_ttm_io_mem_reserve()
271 mem->bus.is_iomem = !rdev->agp->cant_use_aperture; in radeon_ttm_io_mem_reserve()
272 mem->bus.caching = ttm_write_combined; in radeon_ttm_io_mem_reserve()
277 mem->bus.offset = mem->start << PAGE_SHIFT; in radeon_ttm_io_mem_reserve()
279 if ((mem->bus.offset + bus_size) > rdev->mc.visible_vram_size) in radeon_ttm_io_mem_reserve()
280 return -EINVAL; in radeon_ttm_io_mem_reserve()
281 mem->bus.offset += rdev->mc.aper_base; in radeon_ttm_io_mem_reserve()
282 mem->bus.is_iomem = true; in radeon_ttm_io_mem_reserve()
283 mem->bus.caching = ttm_write_combined; in radeon_ttm_io_mem_reserve()
286 * Alpha: use bus.addr to hold the ioremap() return, in radeon_ttm_io_mem_reserve()
287 * so we can modify bus.base below. in radeon_ttm_io_mem_reserve()
289 mem->bus.addr = ioremap_wc(mem->bus.offset, bus_size); in radeon_ttm_io_mem_reserve()
290 if (!mem->bus.addr) in radeon_ttm_io_mem_reserve()
291 return -ENOMEM; in radeon_ttm_io_mem_reserve()
294 * Alpha: Use just the bus offset plus in radeon_ttm_io_mem_reserve()
295 * the hose/domain memory base for bus.base. in radeon_ttm_io_mem_reserve()
299 mem->bus.offset = (mem->bus.offset & 0x0ffffffffUL) + in radeon_ttm_io_mem_reserve()
300 rdev->hose->dense_mem_base; in radeon_ttm_io_mem_reserve()
304 return -EINVAL; in radeon_ttm_io_mem_reserve()
330 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); in radeon_ttm_tt_pin_userptr()
334 if (current->mm != gtt->usermm) in radeon_ttm_tt_pin_userptr()
335 return -EPERM; in radeon_ttm_tt_pin_userptr()
337 if (gtt->userflags & RADEON_GEM_USERPTR_ANONONLY) { in radeon_ttm_tt_pin_userptr()
340 unsigned long end = gtt->userptr + (u64)ttm->num_pages * PAGE_SIZE; in radeon_ttm_tt_pin_userptr()
342 vma = find_vma(gtt->usermm, gtt->userptr); in radeon_ttm_tt_pin_userptr()
343 if (!vma || vma->vm_file || vma->vm_end < end) in radeon_ttm_tt_pin_userptr()
344 return -EPERM; in radeon_ttm_tt_pin_userptr()
348 unsigned num_pages = ttm->num_pages - pinned; in radeon_ttm_tt_pin_userptr()
349 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; in radeon_ttm_tt_pin_userptr()
350 struct page **pages = ttm->pages + pinned; in radeon_ttm_tt_pin_userptr()
359 } while (pinned < ttm->num_pages); in radeon_ttm_tt_pin_userptr()
361 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, in radeon_ttm_tt_pin_userptr()
362 (u64)ttm->num_pages << PAGE_SHIFT, in radeon_ttm_tt_pin_userptr()
367 r = dma_map_sgtable(rdev->dev, ttm->sg, direction, 0); in radeon_ttm_tt_pin_userptr()
371 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in radeon_ttm_tt_pin_userptr()
372 ttm->num_pages); in radeon_ttm_tt_pin_userptr()
377 kfree(ttm->sg); in radeon_ttm_tt_pin_userptr()
380 release_pages(ttm->pages, pinned); in radeon_ttm_tt_pin_userptr()
390 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); in radeon_ttm_tt_unpin_userptr()
395 if (!ttm->sg || !ttm->sg->sgl) in radeon_ttm_tt_unpin_userptr()
399 dma_unmap_sgtable(rdev->dev, ttm->sg, direction, 0); in radeon_ttm_tt_unpin_userptr()
401 for_each_sgtable_page(ttm->sg, &sg_iter, 0) { in radeon_ttm_tt_unpin_userptr()
403 if (!(gtt->userflags & RADEON_GEM_USERPTR_READONLY)) in radeon_ttm_tt_unpin_userptr()
410 sg_free_table(ttm->sg); in radeon_ttm_tt_unpin_userptr()
417 return (gtt->bound); in radeon_ttm_backend_is_bound()
430 if (gtt->bound) in radeon_ttm_backend_bind()
433 if (gtt->userptr) { in radeon_ttm_backend_bind()
438 gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); in radeon_ttm_backend_bind()
439 if (!ttm->num_pages) { in radeon_ttm_backend_bind()
441 ttm->num_pages, bo_mem, ttm); in radeon_ttm_backend_bind()
443 if (ttm->caching == ttm_cached) in radeon_ttm_backend_bind()
445 r = radeon_gart_bind(rdev, gtt->offset, ttm->num_pages, in radeon_ttm_backend_bind()
446 ttm->pages, gtt->ttm.dma_address, flags); in radeon_ttm_backend_bind()
449 ttm->num_pages, (unsigned)gtt->offset); in radeon_ttm_backend_bind()
452 gtt->bound = true; in radeon_ttm_backend_bind()
461 if (gtt->userptr) in radeon_ttm_backend_unbind()
464 if (!gtt->bound) in radeon_ttm_backend_unbind()
467 radeon_gart_unbind(rdev, gtt->offset, ttm->num_pages); in radeon_ttm_backend_unbind()
469 gtt->bound = false; in radeon_ttm_backend_unbind()
476 ttm_tt_fini(>t->ttm); in radeon_ttm_backend_destroy()
487 struct radeon_device *rdev = radeon_get_rdev(bo->bdev); in radeon_ttm_tt_create()
489 if (rdev->flags & RADEON_IS_AGP) { in radeon_ttm_tt_create()
490 return ttm_agp_tt_create(bo, rdev->agp->bridge, page_flags); in radeon_ttm_tt_create()
500 if (rbo->flags & RADEON_GEM_GTT_UC) in radeon_ttm_tt_create()
502 else if (rbo->flags & RADEON_GEM_GTT_WC) in radeon_ttm_tt_create()
507 if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) { in radeon_ttm_tt_create()
511 return >t->ttm; in radeon_ttm_tt_create()
518 if (rdev->flags & RADEON_IS_AGP) in radeon_ttm_tt_to_gtt()
533 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); in radeon_ttm_tt_populate()
535 if (gtt && gtt->userptr) { in radeon_ttm_tt_populate()
536 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); in radeon_ttm_tt_populate()
537 if (!ttm->sg) in radeon_ttm_tt_populate()
538 return -ENOMEM; in radeon_ttm_tt_populate()
540 ttm->page_flags |= TTM_TT_FLAG_EXTERNAL; in radeon_ttm_tt_populate()
544 if (slave && ttm->sg) { in radeon_ttm_tt_populate()
545 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in radeon_ttm_tt_populate()
546 ttm->num_pages); in radeon_ttm_tt_populate()
550 return ttm_pool_alloc(&rdev->mman.bdev.pool, ttm, ctx); in radeon_ttm_tt_populate()
557 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); in radeon_ttm_tt_unpopulate()
561 if (gtt && gtt->userptr) { in radeon_ttm_tt_unpopulate()
562 kfree(ttm->sg); in radeon_ttm_tt_unpopulate()
563 ttm->page_flags &= ~TTM_TT_FLAG_EXTERNAL; in radeon_ttm_tt_unpopulate()
570 return ttm_pool_free(&rdev->mman.bdev.pool, ttm); in radeon_ttm_tt_unpopulate()
580 return -EINVAL; in radeon_ttm_tt_set_userptr()
582 gtt->userptr = addr; in radeon_ttm_tt_set_userptr()
583 gtt->usermm = current->mm; in radeon_ttm_tt_set_userptr()
584 gtt->userflags = flags; in radeon_ttm_tt_set_userptr()
593 if (rdev->flags & RADEON_IS_AGP) in radeon_ttm_tt_is_bound()
608 return -EINVAL; in radeon_ttm_tt_bind()
610 if (rdev->flags & RADEON_IS_AGP) in radeon_ttm_tt_bind()
623 if (rdev->flags & RADEON_IS_AGP) { in radeon_ttm_tt_unbind()
637 if (rdev->flags & RADEON_IS_AGP) { in radeon_ttm_tt_destroy()
653 return !!gtt->userptr; in radeon_ttm_tt_has_userptr()
664 return !!(gtt->userflags & RADEON_GEM_USERPTR_READONLY); in radeon_ttm_tt_is_readonly()
683 r = ttm_device_init(&rdev->mman.bdev, &radeon_bo_driver, rdev->dev, in radeon_ttm_init()
684 rdev_to_drm(rdev)->anon_inode->i_mapping, in radeon_ttm_init()
685 rdev_to_drm(rdev)->vma_offset_manager, in radeon_ttm_init()
686 rdev->need_swiotlb, in radeon_ttm_init()
687 dma_addressing_limited(&rdev->pdev->dev)); in radeon_ttm_init()
692 rdev->mman.initialized = true; in radeon_ttm_init()
700 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); in radeon_ttm_init()
704 NULL, &rdev->stolen_vga_memory); in radeon_ttm_init()
708 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); in radeon_ttm_init()
711 r = radeon_bo_pin(rdev->stolen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); in radeon_ttm_init()
712 radeon_bo_unreserve(rdev->stolen_vga_memory); in radeon_ttm_init()
714 radeon_bo_unref(&rdev->stolen_vga_memory); in radeon_ttm_init()
718 (unsigned) (rdev->mc.real_vram_size / (1024 * 1024))); in radeon_ttm_init()
726 (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); in radeon_ttm_init()
737 if (!rdev->mman.initialized) in radeon_ttm_fini()
740 if (rdev->stolen_vga_memory) { in radeon_ttm_fini()
741 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); in radeon_ttm_fini()
743 radeon_bo_unpin(rdev->stolen_vga_memory); in radeon_ttm_fini()
744 radeon_bo_unreserve(rdev->stolen_vga_memory); in radeon_ttm_fini()
746 radeon_bo_unref(&rdev->stolen_vga_memory); in radeon_ttm_fini()
748 ttm_range_man_fini(&rdev->mman.bdev, TTM_PL_VRAM); in radeon_ttm_fini()
749 ttm_range_man_fini(&rdev->mman.bdev, TTM_PL_TT); in radeon_ttm_fini()
750 ttm_device_fini(&rdev->mman.bdev); in radeon_ttm_fini()
752 rdev->mman.initialized = false; in radeon_ttm_fini()
762 if (!rdev->mman.initialized) in radeon_ttm_set_active_vram_size()
765 man = ttm_manager_type(&rdev->mman.bdev, TTM_PL_VRAM); in radeon_ttm_set_active_vram_size()
767 man->size = size >> PAGE_SHIFT; in radeon_ttm_set_active_vram_size()
774 struct radeon_device *rdev = m->private; in radeon_ttm_page_pool_show()
776 return ttm_pool_debugfs(&rdev->mman.bdev.pool, m); in radeon_ttm_page_pool_show()
783 struct radeon_device *rdev = inode->i_private; in radeon_ttm_vram_open()
784 i_size_write(inode, rdev->mc.mc_vram_size); in radeon_ttm_vram_open()
785 filep->private_data = inode->i_private; in radeon_ttm_vram_open()
792 struct radeon_device *rdev = f->private_data; in radeon_ttm_vram_read()
797 return -EINVAL; in radeon_ttm_vram_read()
803 if (*pos >= rdev->mc.mc_vram_size) in radeon_ttm_vram_read()
806 spin_lock_irqsave(&rdev->mmio_idx_lock, flags); in radeon_ttm_vram_read()
808 if (rdev->family >= CHIP_CEDAR) in radeon_ttm_vram_read()
811 spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); in radeon_ttm_vram_read()
820 size -= 4; in radeon_ttm_vram_read()
835 struct radeon_device *rdev = inode->i_private; in radeon_ttm_gtt_open()
836 i_size_write(inode, rdev->mc.gtt_size); in radeon_ttm_gtt_open()
837 filep->private_data = inode->i_private; in radeon_ttm_gtt_open()
844 struct radeon_device *rdev = f->private_data; in radeon_ttm_gtt_read()
851 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); in radeon_ttm_gtt_read()
855 if (p >= rdev->gart.num_cpu_pages) in radeon_ttm_gtt_read()
858 page = rdev->gart.pages[p]; in radeon_ttm_gtt_read()
869 return -EFAULT; in radeon_ttm_gtt_read()
874 size -= cur_size; in radeon_ttm_gtt_read()
892 struct drm_minor *minor = rdev_to_drm(rdev)->primary; in radeon_ttm_debugfs_init()
893 struct dentry *root = minor->debugfs_root; in radeon_ttm_debugfs_init()
901 ttm_resource_manager_create_debugfs(ttm_manager_type(&rdev->mman.bdev, in radeon_ttm_debugfs_init()
904 ttm_resource_manager_create_debugfs(ttm_manager_type(&rdev->mman.bdev, in radeon_ttm_debugfs_init()