1 /* SPDX-License-Identifier: GPL-2.0 */
17 #include <linux/blk-cgroup.h>
20 #include <linux/blk-mq.h>
28 /* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */
63 	/* all non-root blkcg_gq's are guaranteed to have access to parent */
75 	struct blkg_policy_data		*pd[BLKCG_MAX_POLS];  member
130  * information per blkcg - q pair.
139 	/* the blkg and policy id this per-policy data belongs to */
146  * Policies that need to keep per-blkcg data which is independent from any
150  * cpd_init() is invoked to let each policy handle per-blkcg data.
153 	/* the blkcg and policy id this per-policy data belongs to */
164 typedef void (blkcg_pol_init_pd_fn)(struct blkg_policy_data *pd);
165 typedef void (blkcg_pol_online_pd_fn)(struct blkg_policy_data *pd);
166 typedef void (blkcg_pol_offline_pd_fn)(struct blkg_policy_data *pd);
167 typedef void (blkcg_pol_free_pd_fn)(struct blkg_policy_data *pd);
168 typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkg_policy_data *pd);
169 typedef void (blkcg_pol_stat_pd_fn)(struct blkg_policy_data *pd,
211 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
227  * bio_issue_as_root_blkg - see if this bio needs to be issued as root blkg
241 	return (bio->bi_opf & (REQ_META | REQ_SWAP)) != 0;  in bio_issue_as_root_blkg()
245  * blkg_lookup - lookup blkg for the specified blkcg - q pair
249  * Lookup blkg for the @blkcg - @q pair.
259 		return q->root_blkg;  in blkg_lookup()
261 	blkg = rcu_dereference_check(blkcg->blkg_hint,  in blkg_lookup()
262 			lockdep_is_held(&q->queue_lock));  in blkg_lookup()
263 	if (blkg && blkg->q == q)  in blkg_lookup()
266 	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);  in blkg_lookup()
267 	if (blkg && blkg->q != q)  in blkg_lookup()
273  * blkg_to_pd - get policy private data
277  * Return pointer to private data associated with the @blkg-@pol pair.
282 	return blkg ? blkg->pd[pol->plid] : NULL;  in blkg_to_pd()
288 	return blkcg ? blkcg->cpd[pol->plid] : NULL;  in blkcg_to_cpd()
292  * pd_to_blkg - get blkg associated with policy private data
293  * @pd: policy private data of interest
295  * @pd is policy private data.  Determine the blkg it's associated with.
297 static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)  in pd_to_blkg()  argument
299 	return pd ? pd->blkg : NULL;  in pd_to_blkg()
304 	return cpd ? cpd->blkcg : NULL;  in cpd_to_blkcg()
308  * blkg_get - get a blkg reference
315 	percpu_ref_get(&blkg->refcnt);  in blkg_get()
319  * blkg_tryget - try and get a blkg reference
327 	return blkg && percpu_ref_tryget(&blkg->refcnt);  in blkg_tryget()
331  * blkg_put - put a blkg reference
336 	percpu_ref_put(&blkg->refcnt);  in blkg_put()
340  * blkg_for_each_descendant_pre - pre-order walk of a blkg's descendants
352 	css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css)	\
354 					    (p_blkg)->q)))
357  * blkg_for_each_descendant_post - post-order walk of a blkg's descendants
362  * Similar to blkg_for_each_descendant_pre() but performs post-order
367 	css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css)	\
369 					    (p_blkg)->q)))
373 	bio_issue_init(&bio->bi_issue, bio_sectors(bio));  in blkcg_bio_issue_init()
378 	if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))  in blkcg_use_delay()
380 	if (atomic_add_return(1, &blkg->use_delay) == 1)  in blkcg_use_delay()
381 		atomic_inc(&blkg->blkcg->congestion_count);  in blkcg_use_delay()
386 	int old = atomic_read(&blkg->use_delay);  in blkcg_unuse_delay()
400 	while (old && !atomic_try_cmpxchg(&blkg->use_delay, &old, old - 1))  in blkcg_unuse_delay()
406 		atomic_dec(&blkg->blkcg->congestion_count);  in blkcg_unuse_delay()
411  * blkcg_set_delay - Enable allocator delay mechanism with the specified delay amount
421 	int old = atomic_read(&blkg->use_delay);  in blkcg_set_delay()
424 	if (!old && atomic_try_cmpxchg(&blkg->use_delay, &old, -1))  in blkcg_set_delay()
425 		atomic_inc(&blkg->blkcg->congestion_count);  in blkcg_set_delay()
427 	atomic64_set(&blkg->delay_nsec, delay);  in blkcg_set_delay()
431  * blkcg_clear_delay - Disable allocator delay mechanism
434  * Disable use_delay mechanism. See blkcg_set_delay().
438 	int old = atomic_read(&blkg->use_delay);  in blkcg_clear_delay()
441 	if (old && atomic_try_cmpxchg(&blkg->use_delay, &old, 0))  in blkcg_clear_delay()
442 		atomic_dec(&blkg->blkcg->congestion_count);  in blkcg_clear_delay()
446  * blk_cgroup_mergeable - Determine whether to allow or disallow merges
456 	return rq->bio->bi_blkg == bio->bi_blkg &&  in blk_cgroup_mergeable()
457 		bio_issue_as_root_blkg(rq->bio) == bio_issue_as_root_blkg(bio);  in blk_cgroup_mergeable()
489 static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }  in pd_to_blkg()  argument
497 	for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)