78 static inline struct folio *try_get_folio(struct page *page, int refs)  in try_get_folio()  argument
86 	if (unlikely(!folio_ref_try_add(folio, refs)))  in try_get_folio()
99 		if (!put_devmap_managed_folio_refs(folio, refs))  in try_get_folio()
100 			folio_put_refs(folio, refs);  in try_get_folio()
107 static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)  in gup_put_folio()  argument
112 		node_stat_mod_folio(folio, NR_FOLL_PIN_RELEASED, refs);  in gup_put_folio()
114 			atomic_sub(refs, &folio->_pincount);  in gup_put_folio()
116 			refs *= GUP_PIN_COUNTING_BIAS;  in gup_put_folio()
119 	if (!put_devmap_managed_folio_refs(folio, refs))  in gup_put_folio()
120 		folio_put_refs(folio, refs);  in gup_put_folio()
126  * @refs:     the value to (effectively) add to the folio's refcount
146 int __must_check try_grab_folio(struct folio *folio, int refs,  in try_grab_folio()  argument
156 		folio_ref_add(folio, refs);  in try_grab_folio()
170 			folio_ref_add(folio, refs);  in try_grab_folio()
171 			atomic_add(refs, &folio->_pincount);  in try_grab_folio()
173 			folio_ref_add(folio, refs * GUP_PIN_COUNTING_BIAS);  in try_grab_folio()
176 		node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);  in try_grab_folio()
505  * @refs:  the value to (effectively) add to the folio's refcount
515  *    FOLL_GET: folio's refcount will be incremented by @refs.
518  *    @refs, and its pincount will be incremented by @refs.
521  *    @refs * GUP_PIN_COUNTING_BIAS.
531 static struct folio *try_grab_folio_fast(struct page *page, int refs,  in try_grab_folio_fast()  argument
546 		return try_get_folio(page, refs);  in try_grab_folio_fast()
557 	folio = try_get_folio(page, refs);  in try_grab_folio_fast()
568 		if (!put_devmap_managed_folio_refs(folio, refs))  in try_grab_folio_fast()
569 			folio_put_refs(folio, refs);  in try_grab_folio_fast()
582 		atomic_add(refs, &folio->_pincount);  in try_grab_folio_fast()
585 				refs * (GUP_PIN_COUNTING_BIAS - 1));  in try_grab_folio_fast()
593 	node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);  in try_grab_folio_fast()
3095 	int refs;  in gup_fast_pmd_leaf()  local
3111 	refs = record_subpages(page, PMD_SIZE, addr, end, pages + *nr);  in gup_fast_pmd_leaf()
3113 	folio = try_grab_folio_fast(page, refs, flags);  in gup_fast_pmd_leaf()
3118 		gup_put_folio(folio, refs, flags);  in gup_fast_pmd_leaf()
3123 		gup_put_folio(folio, refs, flags);  in gup_fast_pmd_leaf()
3127 		gup_put_folio(folio, refs, flags);  in gup_fast_pmd_leaf()
3131 	*nr += refs;  in gup_fast_pmd_leaf()
3142 	int refs;  in gup_fast_pud_leaf()  local
3158 	refs = record_subpages(page, PUD_SIZE, addr, end, pages + *nr);  in gup_fast_pud_leaf()
3160 	folio = try_grab_folio_fast(page, refs, flags);  in gup_fast_pud_leaf()
3165 		gup_put_folio(folio, refs, flags);  in gup_fast_pud_leaf()
3170 		gup_put_folio(folio, refs, flags);  in gup_fast_pud_leaf()
3175 		gup_put_folio(folio, refs, flags);  in gup_fast_pud_leaf()
3179 	*nr += refs;  in gup_fast_pud_leaf()
3188 	int refs;  in gup_fast_pgd_leaf()  local
3198 	refs = record_subpages(page, PGDIR_SIZE, addr, end, pages + *nr);  in gup_fast_pgd_leaf()
3200 	folio = try_grab_folio_fast(page, refs, flags);  in gup_fast_pgd_leaf()
3205 		gup_put_folio(folio, refs, flags);  in gup_fast_pgd_leaf()
3210 		gup_put_folio(folio, refs, flags);  in gup_fast_pgd_leaf()
3215 		gup_put_folio(folio, refs, flags);  in gup_fast_pgd_leaf()
3219 	*nr += refs;  in gup_fast_pgd_leaf()