51 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
53 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
60 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_has_pending()
70 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,  in blk_mq_hctx_mark_pending()
79 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,  in blk_mq_hctx_clear_pending()
374 	struct blk_mq_hw_ctx *hctx;  in blk_mq_wake_waiters()
413 	struct blk_mq_hw_ctx *hctx = data->hctx;  in blk_mq_rq_ctx_init()
769 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in __blk_mq_free_request()
1153 static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,  in blk_mq_flush_tag_batch()
1167 	struct blk_mq_hw_ctx *cur_hctx = NULL;  in blk_mq_end_request_batch()
1415 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_execute_rq_nowait()
1476 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_execute_rq()
1708 	struct blk_mq_hw_ctx *hctx;  in blk_mq_timeout_work()
1761 	struct blk_mq_hw_ctx *hctx;
1768 	struct blk_mq_hw_ctx *hctx = flush_data->hctx;  in flush_busy_ctx()
1783 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)  in blk_mq_flush_busy_ctxs()
1794 	struct blk_mq_hw_ctx *hctx;
1802 	struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;  in dispatch_rq_from_ctx()
1818 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,  in blk_mq_dequeue_from_ctx()
1861 	struct blk_mq_hw_ctx *hctx;  in blk_mq_dispatch_wake()
1863 	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);  in blk_mq_dispatch_wake()
1885 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,  in blk_mq_mark_tag_wait()
1979 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)  in blk_mq_update_dispatch_busy()
2014 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_prep_dispatch_rq()
2070 static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued,  in blk_mq_commit_rqs()
2082 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,  in blk_mq_dispatch_rq_list()
2212 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)  in blk_mq_first_mapped_cpu()
2225 static bool blk_mq_hctx_empty_cpumask(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_empty_cpumask()
2236 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_next_cpu()
2284 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)  in blk_mq_delay_run_hw_queue()
2293 static inline bool blk_mq_hw_queue_need_run(struct blk_mq_hw_ctx *hctx)  in blk_mq_hw_queue_need_run()
2320 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)  in blk_mq_run_hw_queue()
2363 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)  in blk_mq_get_sq_hctx()
2373 	struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT];  in blk_mq_get_sq_hctx()
2387 	struct blk_mq_hw_ctx *hctx, *sq_hctx;  in blk_mq_run_hw_queues()
2415 	struct blk_mq_hw_ctx *hctx, *sq_hctx;  in blk_mq_delay_run_hw_queues()
2453 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)  in blk_mq_stop_hw_queue()
2472 	struct blk_mq_hw_ctx *hctx;  in blk_mq_stop_hw_queues()
2480 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)  in blk_mq_start_hw_queue()
2490 	struct blk_mq_hw_ctx *hctx;  in blk_mq_start_hw_queues()
2498 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)  in blk_mq_start_stopped_hw_queue()
2516 	struct blk_mq_hw_ctx *hctx;  in blk_mq_start_stopped_hw_queues()
2527 	struct blk_mq_hw_ctx *hctx =  in blk_mq_run_work_fn()
2528 		container_of(work, struct blk_mq_hw_ctx, run_work.work);  in blk_mq_run_work_fn()
2544 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_request_bypass_insert()
2554 static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,  in blk_mq_insert_requests()
2595 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_insert_request()
2676 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,  in __blk_mq_issue_directly()
2734 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,  in blk_mq_try_issue_directly()
2768 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_request_issue_directly()
2783 	struct blk_mq_hw_ctx *hctx = NULL;  in blk_mq_plug_issue_direct()
2830 	struct blk_mq_hw_ctx *this_hctx = NULL;  in blk_mq_dispatch_plug_list()
2921 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,  in blk_mq_try_issue_list_directly()
3061 	struct blk_mq_hw_ctx *hctx;  in blk_mq_submit_bio()
3605 	struct blk_mq_hw_ctx *hctx;
3619 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_has_requests()
3631 static bool blk_mq_hctx_has_online_cpu(struct blk_mq_hw_ctx *hctx,  in blk_mq_hctx_has_online_cpu()
3643 		struct blk_mq_hw_ctx *h = blk_mq_map_queue_type(hctx->queue,  in blk_mq_hctx_has_online_cpu()
3659 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,  in blk_mq_hctx_notify_offline()
3660 			struct blk_mq_hw_ctx, cpuhp_online);  in blk_mq_hctx_notify_offline()
3697 		const struct blk_mq_hw_ctx *hctx)  in blk_mq_cpu_mapped_to_hctx()
3699 	struct blk_mq_hw_ctx *mapped_hctx = blk_mq_map_queue_type(hctx->queue,  in blk_mq_cpu_mapped_to_hctx()
3707 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,  in blk_mq_hctx_notify_online()
3708 			struct blk_mq_hw_ctx, cpuhp_online);  in blk_mq_hctx_notify_online()
3722 	struct blk_mq_hw_ctx *hctx;  in blk_mq_hctx_notify_dead()
3727 	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);  in blk_mq_hctx_notify_dead()
3752 static void __blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)  in __blk_mq_remove_cpuhp()
3770 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)  in blk_mq_remove_cpuhp()
3777 static void __blk_mq_add_cpuhp(struct blk_mq_hw_ctx *hctx)  in __blk_mq_add_cpuhp()
3793 	struct blk_mq_hw_ctx *hctx;  in __blk_mq_remove_cpuhp_list()
3830 	struct blk_mq_hw_ctx *hctx;  in blk_mq_add_hw_queues_cpuhp()
3871 		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)  in blk_mq_exit_hctx()
3897 	struct blk_mq_hw_ctx *hctx;  in blk_mq_exit_hw_queues()
3910 		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)  in blk_mq_init_hctx()
3939 static struct blk_mq_hw_ctx *
3943 	struct blk_mq_hw_ctx *hctx;  in blk_mq_alloc_hctx()
3946 	hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node);  in blk_mq_alloc_hctx()
4014 		struct blk_mq_hw_ctx *hctx;  in blk_mq_init_cpu_queues()
4094 	struct blk_mq_hw_ctx *hctx;  in blk_mq_map_swqueue()
4210 	struct blk_mq_hw_ctx *hctx;  in queue_set_hctx_shared()
4311 	struct blk_mq_hw_ctx *hctx, *next;  in blk_mq_release()
4424 static bool blk_mq_hctx_is_reusable(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_is_reusable()
4430 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(  in blk_mq_alloc_and_init_hctx()
4434 	struct blk_mq_hw_ctx *hctx = NULL, *tmp;  in blk_mq_alloc_and_init_hctx()
4467 	struct blk_mq_hw_ctx *hctx;  in blk_mq_realloc_hw_ctxs()
4475 		struct blk_mq_hw_ctx *old_hctx = xa_load(&q->hctx_table, i);  in blk_mq_realloc_hw_ctxs()
4863 	struct blk_mq_hw_ctx *hctx;  in blk_mq_update_nr_requests()
5076 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,  in blk_hctx_poll()
5137 	struct blk_mq_hw_ctx *hctx;  in blk_mq_cancel_work_sync()