2 /* Page fragment allocator
4  * Page Fragment:
6  *  0 or higher order page.  Multiple fragments within that page are
7  *  individually refcounted, in the page's reference counter.
9  * The page_frag functions provide a simple allocation framework for page
23 static unsigned long encoded_page_create(struct page *page, unsigned int order,  in encoded_page_create()  argument
29 	return (unsigned long)page_address(page) |  in encoded_page_create()
44 static struct page *encoded_page_decode_page(unsigned long encoded_page)  in encoded_page_decode_page()
49 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,  in __page_frag_cache_refill()
53 	struct page *page = NULL;  in __page_frag_cache_refill()  local
59 	page = __alloc_pages(gfp_mask, PAGE_FRAG_CACHE_MAX_ORDER,  in __page_frag_cache_refill()
62 	if (unlikely(!page)) {  in __page_frag_cache_refill()
63 		page = __alloc_pages(gfp, 0, numa_mem_id(), NULL);  in __page_frag_cache_refill()
67 	nc->encoded_page = page ?  in __page_frag_cache_refill()
68 		encoded_page_create(page, order, page_is_pfmemalloc(page)) : 0;  in __page_frag_cache_refill()
70 	return page;  in __page_frag_cache_refill()
84 void __page_frag_cache_drain(struct page *page, unsigned int count)  in __page_frag_cache_drain()  argument
86 	VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);  in __page_frag_cache_drain()
88 	if (page_ref_sub_and_test(page, count))  in __page_frag_cache_drain()
89 		free_frozen_pages(page, compound_order(page));  in __page_frag_cache_drain()
99 	struct page *page;  in __page_frag_alloc_align()  local
103 		page = __page_frag_cache_refill(nc, gfp_mask);  in __page_frag_alloc_align()
104 		if (!page)  in __page_frag_alloc_align()
109 		/* Even if we own the page, we do not use atomic_set().  in __page_frag_alloc_align()
112 		page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);  in __page_frag_alloc_align()
114 		/* reset page count bias and offset to start of new frag */  in __page_frag_alloc_align()
128 			 * We don't release the cache page because  in __page_frag_alloc_align()
135 		page = encoded_page_decode_page(encoded_page);  in __page_frag_alloc_align()
137 		if (!page_ref_sub_and_test(page, nc->pagecnt_bias))  in __page_frag_alloc_align()
141 			free_frozen_pages(page,  in __page_frag_alloc_align()
146 		/* OK, page count is 0, we can safely set it */  in __page_frag_alloc_align()
147 		set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);  in __page_frag_alloc_align()
149 		/* reset page count bias and offset to start of new frag */  in __page_frag_alloc_align()
162  * Frees a page fragment allocated out of either a compound or order 0 page.
166 	struct page *page = virt_to_head_page(addr);  in page_frag_free()  local
168 	if (unlikely(put_page_testzero(page)))  in page_frag_free()
169 		free_frozen_pages(page, compound_order(page));  in page_frag_free()