27 #include <linux/migrate.h>
181  * @mfence: migrate fence
263 static unsigned long svm_migrate_unsuccessful_pages(struct migrate_vma *migrate)  in svm_migrate_unsuccessful_pages()  argument
268 	for (i = 0; i < migrate->npages; i++) {  in svm_migrate_unsuccessful_pages()
269 		if (migrate->src[i] & MIGRATE_PFN_VALID &&  in svm_migrate_unsuccessful_pages()
270 		    !(migrate->src[i] & MIGRATE_PFN_MIGRATE))  in svm_migrate_unsuccessful_pages()
278 			 struct migrate_vma *migrate, struct dma_fence **mfence,  in svm_migrate_copy_to_vram()  argument
281 	uint64_t npages = migrate->npages;  in svm_migrate_copy_to_vram()
299 	for (i = j = 0; (i < npages) && (mpages < migrate->cpages); i++) {  in svm_migrate_copy_to_vram()
302 		if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {  in svm_migrate_copy_to_vram()
304 			migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);  in svm_migrate_copy_to_vram()
305 			svm_migrate_get_vram_page(prange, migrate->dst[i]);  in svm_migrate_copy_to_vram()
306 			migrate->dst[i] = migrate_pfn(migrate->dst[i]);  in svm_migrate_copy_to_vram()
309 		spage = migrate_pfn_to_page(migrate->src[i]);  in svm_migrate_copy_to_vram()
363 			migrate->dst[i] = 0;  in svm_migrate_copy_to_vram()
373 		migrate->dst[i] = 0;  in svm_migrate_copy_to_vram()
375 		migrate->dst[i + 1] = 0;  in svm_migrate_copy_to_vram()
377 		migrate->dst[i + 2] = 0;  in svm_migrate_copy_to_vram()
379 		migrate->dst[i + 3] = 0;  in svm_migrate_copy_to_vram()
396 	struct migrate_vma migrate = { 0 };  in svm_migrate_vma_to_vram()  local
403 	memset(&migrate, 0, sizeof(migrate));  in svm_migrate_vma_to_vram()
404 	migrate.vma = vma;  in svm_migrate_vma_to_vram()
405 	migrate.start = start;  in svm_migrate_vma_to_vram()
406 	migrate.end = end;  in svm_migrate_vma_to_vram()
407 	migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;  in svm_migrate_vma_to_vram()
408 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);  in svm_migrate_vma_to_vram()
411 		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),  in svm_migrate_vma_to_vram()
416 	migrate.src = buf;  in svm_migrate_vma_to_vram()
417 	migrate.dst = migrate.src + npages;  in svm_migrate_vma_to_vram()
418 	scratch = (dma_addr_t *)(migrate.dst + npages);  in svm_migrate_vma_to_vram()
425 	r = migrate_vma_setup(&migrate);  in svm_migrate_vma_to_vram()
432 	cpages = migrate.cpages;  in svm_migrate_vma_to_vram()
434 		pr_debug("failed collect migrate sys pages [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_vram()
444 	r = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);  in svm_migrate_vma_to_vram()
445 	migrate_vma_pages(&migrate);  in svm_migrate_vma_to_vram()
448 	migrate_vma_finalize(&migrate);  in svm_migrate_vma_to_vram()
450 	mpages = cpages - svm_migrate_unsuccessful_pages(&migrate);  in svm_migrate_vma_to_vram()
452 			 mpages, cpages, migrate.npages);  in svm_migrate_vma_to_vram()
473  * svm_migrate_ram_to_vram - migrate svm range from system to device
475  * @best_loc: the device to migrate to
476  * @start_mgr: start page to migrate
477  * @last_mgr: last page to migrate
543 			pr_debug("failed %ld to migrate\n", r);  in svm_migrate_ram_to_vram()
582 			struct migrate_vma *migrate, struct dma_fence **mfence,  in svm_migrate_copy_to_ram()  argument
596 	addr = migrate->start;  in svm_migrate_copy_to_ram()
604 		spage = migrate_pfn_to_page(migrate->src[i]);  in svm_migrate_copy_to_ram()
630 		dpage = svm_migrate_get_sys_page(migrate->vma, addr);  in svm_migrate_copy_to_ram()
648 		migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));  in svm_migrate_copy_to_ram()
660 			migrate->dst[i] = 0;  in svm_migrate_copy_to_ram()
668  * svm_migrate_vma_to_ram - migrate range inside one vma from device to system
674  * @node: kfd node device to migrate from
697 	struct migrate_vma migrate = { 0 };  in svm_migrate_vma_to_ram()  local
702 	memset(&migrate, 0, sizeof(migrate));  in svm_migrate_vma_to_ram()
703 	migrate.vma = vma;  in svm_migrate_vma_to_ram()
704 	migrate.start = start;  in svm_migrate_vma_to_ram()
705 	migrate.end = end;  in svm_migrate_vma_to_ram()
706 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);  in svm_migrate_vma_to_ram()
708 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_COHERENT;  in svm_migrate_vma_to_ram()
710 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;  in svm_migrate_vma_to_ram()
713 		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),  in svm_migrate_vma_to_ram()
718 	migrate.src = buf;  in svm_migrate_vma_to_ram()
719 	migrate.dst = migrate.src + npages;  in svm_migrate_vma_to_ram()
720 	migrate.fault_page = fault_page;  in svm_migrate_vma_to_ram()
721 	scratch = (dma_addr_t *)(migrate.dst + npages);  in svm_migrate_vma_to_ram()
728 	r = migrate_vma_setup(&migrate);  in svm_migrate_vma_to_ram()
735 	cpages = migrate.cpages;  in svm_migrate_vma_to_ram()
737 		pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n",  in svm_migrate_vma_to_ram()
739 		upages = svm_migrate_unsuccessful_pages(&migrate);  in svm_migrate_vma_to_ram()
748 	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,  in svm_migrate_vma_to_ram()
750 	migrate_vma_pages(&migrate);  in svm_migrate_vma_to_ram()
752 	upages = svm_migrate_unsuccessful_pages(&migrate);  in svm_migrate_vma_to_ram()
754 		 upages, cpages, migrate.npages);  in svm_migrate_vma_to_ram()
757 	migrate_vma_finalize(&migrate);  in svm_migrate_vma_to_ram()
778  * svm_migrate_vram_to_ram - migrate svm range from device to system
803 	/* this pragne has no any vram page to migrate to sys ram */  in svm_migrate_vram_to_ram()
842 			pr_debug("failed %ld to migrate prange %p\n", r, prange);  in svm_migrate_vram_to_ram()
866  * svm_migrate_vram_to_vram - migrate svm range from device to device
868  * @best_loc: the device to migrate to
876  * migrate all vram pages in prange to sys ram, then migrate
995 		pr_debug("failed %d migrate svms 0x%p range 0x%p [0x%lx 0x%lx]\n",  in svm_migrate_to_ram()