9 #include <linux/migrate.h>
24 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_skip()  local
28 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_skip()
29 		migrate->src[migrate->npages++] = 0;  in migrate_vma_collect_skip()
40 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_hole()  local
48 		migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;  in migrate_vma_collect_hole()
49 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_hole()
50 		migrate->npages++;  in migrate_vma_collect_hole()
51 		migrate->cpages++;  in migrate_vma_collect_hole()
62 	struct migrate_vma *migrate = walk->private;  in migrate_vma_collect_pmd()  local
120 				migrate->cpages++;  in migrate_vma_collect_pmd()
136 			if (!(migrate->flags &  in migrate_vma_collect_pmd()
138 			    page->pgmap->owner != migrate->pgmap_owner)  in migrate_vma_collect_pmd()
148 			    (migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) {  in migrate_vma_collect_pmd()
150 				migrate->cpages++;  in migrate_vma_collect_pmd()
153 			page = vm_normal_page(migrate->vma, addr, pte);  in migrate_vma_collect_pmd()
155 			    !(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))  in migrate_vma_collect_pmd()
158 			    (!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_COHERENT) ||  in migrate_vma_collect_pmd()
159 			     page->pgmap->owner != migrate->pgmap_owner))  in migrate_vma_collect_pmd()
216 			migrate->cpages++;  in migrate_vma_collect_pmd()
268 		migrate->dst[migrate->npages] = 0;  in migrate_vma_collect_pmd()
269 		migrate->src[migrate->npages++] = mpfn;  in migrate_vma_collect_pmd()
290  * @migrate: migrate struct containing all migration information
296 static void migrate_vma_collect(struct migrate_vma *migrate)  in migrate_vma_collect()  argument
306 		migrate->vma->vm_mm, migrate->start, migrate->end,  in migrate_vma_collect()
307 		migrate->pgmap_owner);  in migrate_vma_collect()
310 	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,  in migrate_vma_collect()
311 			&migrate_vma_walk_ops, migrate);  in migrate_vma_collect()
314 	migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);  in migrate_vma_collect()
440  * @migrate: migrate struct containing all migration information
444  * restored because we cannot migrate them.
449 static void migrate_vma_unmap(struct migrate_vma *migrate)  in migrate_vma_unmap()  argument
451 	migrate->cpages = migrate_device_unmap(migrate->src, migrate->npages,  in migrate_vma_unmap()
452 					migrate->fault_page);  in migrate_vma_unmap()
456  * migrate_vma_setup() - prepare to migrate a range of memory
462  * Prepare to migrate a range of memory virtual address range by collecting all
477  * Note that the caller does not have to migrate all the pages that are marked
479  * device memory to system memory.  If the caller cannot migrate a device page
496  * then migrate_vma_pages() to migrate struct page information from the source
497  * struct page to the destination struct page.  If it fails to migrate the
567 static void migrate_vma_insert_page(struct migrate_vma *migrate,  in migrate_vma_insert_page()  argument
573 	struct vm_area_struct *vma = migrate->vma;  in migrate_vma_insert_page()
685 				struct migrate_vma *migrate)  in __migrate_device_pages()  argument
714 			VM_BUG_ON(!migrate);  in __migrate_device_pages()
715 			addr = migrate->start + i*PAGE_SIZE;  in __migrate_device_pages()
721 					migrate->vma->vm_mm, addr, migrate->end,  in __migrate_device_pages()
722 					migrate->pgmap_owner);  in __migrate_device_pages()
725 			migrate_vma_insert_page(migrate, addr, newpage,  in __migrate_device_pages()
759 		if (migrate && migrate->fault_page == page)  in __migrate_device_pages()
773  * migrate_device_pages() - migrate meta-data from src page to dst page
775  * @dst_pfns: array of pfns allocated by the driver to migrate memory to
778  * Equivalent to migrate_vma_pages(). This is called to migrate struct page
789  * migrate_vma_pages() - migrate meta-data from src page to dst page
790  * @migrate: migrate struct containing all migration information
796 void migrate_vma_pages(struct migrate_vma *migrate)  in migrate_vma_pages()  argument
798 	__migrate_device_pages(migrate->src, migrate->dst, migrate->npages, migrate);  in migrate_vma_pages()
805  * @dst_pfns: array of pfns allocated by the driver to migrate memory to
859  * @migrate: migrate struct containing all migration information
868 void migrate_vma_finalize(struct migrate_vma *migrate)  in migrate_vma_finalize()  argument
870 	migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);  in migrate_vma_finalize()
875  * migrate_device_range() - migrate device private pfns to normal memory.
877  * @start: starting pfn in the range to migrate.
878  * @npages: number of pages to migrate.
923  * Migrate a device coherent folio back to normal memory. The caller should have