75 static void	xprt_request_init(struct rpc_task *task);
258  * @task: task that is requesting access to the transport
265 int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_reserve_xprt()  argument
267 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_reserve_xprt()
270 		if (task == xprt->snd_task)  in xprt_reserve_xprt()
276 	xprt->snd_task = task;  in xprt_reserve_xprt()
279 	trace_xprt_reserve_xprt(xprt, task);  in xprt_reserve_xprt()
285 	task->tk_status = -EAGAIN;  in xprt_reserve_xprt()
286 	if (RPC_IS_SOFT(task) || RPC_IS_SOFTCONN(task))  in xprt_reserve_xprt()
287 		rpc_sleep_on_timeout(&xprt->sending, task, NULL,  in xprt_reserve_xprt()
290 		rpc_sleep_on(&xprt->sending, task, NULL);  in xprt_reserve_xprt()
322  * @task: task that is requesting access to the transport
329 int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_reserve_xprt_cong()  argument
331 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_reserve_xprt_cong()
334 		if (task == xprt->snd_task)  in xprt_reserve_xprt_cong()
339 		xprt->snd_task = task;  in xprt_reserve_xprt_cong()
345 		xprt->snd_task = task;  in xprt_reserve_xprt_cong()
351 	task->tk_status = -EAGAIN;  in xprt_reserve_xprt_cong()
352 	if (RPC_IS_SOFT(task) || RPC_IS_SOFTCONN(task))  in xprt_reserve_xprt_cong()
353 		rpc_sleep_on_timeout(&xprt->sending, task, NULL,  in xprt_reserve_xprt_cong()
356 		rpc_sleep_on(&xprt->sending, task, NULL);  in xprt_reserve_xprt_cong()
359 	trace_xprt_reserve_cong(xprt, task);  in xprt_reserve_xprt_cong()
364 static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_lock_write()  argument
368 	if (test_bit(XPRT_LOCKED, &xprt->state) && xprt->snd_task == task)  in xprt_lock_write()
371 	retval = xprt->ops->reserve_xprt(xprt, task);  in xprt_lock_write()
376 static bool __xprt_lock_write_func(struct rpc_task *task, void *data)  in __xprt_lock_write_func()  argument
380 	xprt->snd_task = task;  in __xprt_lock_write_func()
415  * @task: task that is releasing access to the transport
417  * Note that "task" can be NULL.  No congestion control is provided.
419 void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_xprt()  argument
421 	if (xprt->snd_task == task) {  in xprt_release_xprt()
425 	trace_xprt_release_xprt(xprt, task);  in xprt_release_xprt()
432  * @task: task that is releasing access to the transport
434  * Note that "task" can be NULL.  Another task is awoken to use the
437 void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_xprt_cong()  argument
439 	if (xprt->snd_task == task) {  in xprt_release_xprt_cong()
443 	trace_xprt_release_cong(xprt, task);  in xprt_release_xprt_cong()
447 void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_write()  argument
449 	if (xprt->snd_task != task)  in xprt_release_write()
452 	xprt->ops->release_xprt(xprt, task);  in xprt_release_write()
458  * overflowed. Put the task to sleep if this is the case.
476  * Adjust the congestion window, and wake up the next task
514  * @task: RPC request that recently completed
518 void xprt_release_rqst_cong(struct rpc_task *task)  in xprt_release_rqst_cong()  argument
520 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_release_rqst_cong()
549  * @task: recently completed RPC request used to adjust window
562 void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result)  in xprt_adjust_cwnd()  argument
564 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_adjust_cwnd()
589  * @status: result code to plant in each task before waking it
620 		dprintk("RPC:       write space: waking waiting task on "  in xprt_clear_write_space_locked()
628  * xprt_write_space - wake the task waiting for transport output buffer space
679 static void xprt_init_majortimeo(struct rpc_task *task, struct rpc_rqst *req,  in xprt_init_majortimeo()  argument
688 		time_init = xprt_abs_ktime_to_jiffies(task->tk_start);  in xprt_init_majortimeo()
806 xprt_request_retransmit_after_disconnect(struct rpc_task *task)  in xprt_request_retransmit_after_disconnect()  argument
808 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_retransmit_after_disconnect()
882 		struct rpc_task *task,  in xprt_lock_connect()  argument
890 	if (xprt->snd_task != task)  in xprt_lock_connect()
920  * @task: RPC task that is requesting the connect
923 void xprt_connect(struct rpc_task *task)  in xprt_connect()  argument
925 	struct rpc_xprt	*xprt = task->tk_rqstp->rq_xprt;  in xprt_connect()
930 		task->tk_status = -EAGAIN;  in xprt_connect()
933 	if (!xprt_lock_write(xprt, task))  in xprt_connect()
937 		task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie;  in xprt_connect()
938 		rpc_sleep_on_timeout(&xprt->pending, task, NULL,  in xprt_connect()
939 				xprt_request_timeout(task->tk_rqstp));  in xprt_connect()
948 			xprt->ops->connect(xprt, task);  in xprt_connect()
951 			task->tk_status = 0;  in xprt_connect()
952 			rpc_wake_up_queued_task(&xprt->pending, task);  in xprt_connect()
955 	xprt_release_write(xprt, task);  in xprt_connect()
1127 xprt_request_data_received(struct rpc_task *task)  in xprt_request_data_received()  argument
1129 	return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) &&  in xprt_request_data_received()
1130 		READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) != 0;  in xprt_request_data_received()
1134 xprt_request_need_enqueue_receive(struct rpc_task *task, struct rpc_rqst *req)  in xprt_request_need_enqueue_receive()  argument
1136 	return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) &&  in xprt_request_need_enqueue_receive()
1137 		READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) == 0;  in xprt_request_need_enqueue_receive()
1142  * @task: RPC task
1146 xprt_request_enqueue_receive(struct rpc_task *task)  in xprt_request_enqueue_receive()  argument
1148 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_enqueue_receive()
1152 	if (!xprt_request_need_enqueue_receive(task, req))  in xprt_request_enqueue_receive()
1155 	ret = xprt_request_prepare(task->tk_rqstp, &req->rq_rcv_buf);  in xprt_request_enqueue_receive()
1166 	set_bit(RPC_TASK_NEED_RECV, &task->tk_runstate);  in xprt_request_enqueue_receive()
1176  * @task: RPC task
1181 xprt_request_dequeue_receive_locked(struct rpc_task *task)  in xprt_request_dequeue_receive_locked()  argument
1183 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_dequeue_receive_locked()
1185 	if (test_and_clear_bit(RPC_TASK_NEED_RECV, &task->tk_runstate))  in xprt_request_dequeue_receive_locked()
1191  * @task: RPC request that recently completed
1195 void xprt_update_rtt(struct rpc_task *task)  in xprt_update_rtt()  argument
1197 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_update_rtt()
1198 	struct rpc_rtt *rtt = task->tk_client->cl_rtt;  in xprt_update_rtt()
1199 	unsigned int timer = task->tk_msg.rpc_proc->p_timer;  in xprt_update_rtt()
1212  * @task: RPC request that recently completed
1217 void xprt_complete_rqst(struct rpc_task *task, int copied)  in xprt_complete_rqst()  argument
1219 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_complete_rqst()
1231 	xprt_request_dequeue_receive_locked(task);  in xprt_complete_rqst()
1232 	rpc_wake_up_queued_task(&xprt->pending, task);  in xprt_complete_rqst()
1236 static void xprt_timer(struct rpc_task *task)  in xprt_timer()  argument
1238 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_timer()
1241 	if (task->tk_status != -ETIMEDOUT)  in xprt_timer()
1244 	trace_xprt_timer(xprt, req->rq_xid, task->tk_status);  in xprt_timer()
1247 			xprt->ops->timer(xprt, task);  in xprt_timer()
1249 		task->tk_status = 0;  in xprt_timer()
1254  * @task: pointer to rpc_task
1259  * and put the task to sleep on the pending queue.
1261 void xprt_wait_for_reply_request_def(struct rpc_task *task)  in xprt_wait_for_reply_request_def()  argument
1263 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_wait_for_reply_request_def()
1265 	rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer,  in xprt_wait_for_reply_request_def()
1272  * @task: pointer to rpc_task
1275  * and put the task to sleep on the pending queue.
1277 void xprt_wait_for_reply_request_rtt(struct rpc_task *task)  in xprt_wait_for_reply_request_rtt()  argument
1279 	int timer = task->tk_msg.rpc_proc->p_timer;  in xprt_wait_for_reply_request_rtt()
1280 	struct rpc_clnt *clnt = task->tk_client;  in xprt_wait_for_reply_request_rtt()
1282 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_wait_for_reply_request_rtt()
1290 	rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer,  in xprt_wait_for_reply_request_rtt()
1297  * @task: RPC task about to send a request
1300 void xprt_request_wait_receive(struct rpc_task *task)  in xprt_request_wait_receive()  argument
1302 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_wait_receive()
1305 	if (!test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate))  in xprt_request_wait_receive()
1313 	if (test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) {  in xprt_request_wait_receive()
1314 		xprt->ops->wait_for_reply_request(task);  in xprt_request_wait_receive()
1320 		if (xprt_request_retransmit_after_disconnect(task))  in xprt_request_wait_receive()
1322 					task, -ENOTCONN);  in xprt_request_wait_receive()
1328 xprt_request_need_enqueue_transmit(struct rpc_task *task, struct rpc_rqst *req)  in xprt_request_need_enqueue_transmit()  argument
1330 	return !test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate);  in xprt_request_need_enqueue_transmit()
1334  * xprt_request_enqueue_transmit - queue a task for transmission
1335  * @task: pointer to rpc_task
1337  * Add a task to the transmission queue.
1340 xprt_request_enqueue_transmit(struct rpc_task *task)  in xprt_request_enqueue_transmit()  argument
1342 	struct rpc_rqst *pos, *req = task->tk_rqstp;  in xprt_request_enqueue_transmit()
1346 	if (xprt_request_need_enqueue_transmit(task, req)) {  in xprt_request_enqueue_transmit()
1347 		ret = xprt_request_prepare(task->tk_rqstp, &req->rq_snd_buf);  in xprt_request_enqueue_transmit()
1349 			task->tk_status = ret;  in xprt_request_enqueue_transmit()
1370 				if (pos->rq_task->tk_owner != task->tk_owner)  in xprt_request_enqueue_transmit()
1381 		set_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate);  in xprt_request_enqueue_transmit()
1387  * xprt_request_dequeue_transmit_locked - remove a task from the transmission queue
1388  * @task: pointer to rpc_task
1390  * Remove a task from the transmission queue
1394 xprt_request_dequeue_transmit_locked(struct rpc_task *task)  in xprt_request_dequeue_transmit_locked()  argument
1396 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_dequeue_transmit_locked()
1398 	if (!test_and_clear_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_request_dequeue_transmit_locked()
1421  * xprt_request_dequeue_transmit - remove a task from the transmission queue
1422  * @task: pointer to rpc_task
1424  * Remove a task from the transmission queue
1427 xprt_request_dequeue_transmit(struct rpc_task *task)  in xprt_request_dequeue_transmit()  argument
1429 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_dequeue_transmit()
1433 	xprt_request_dequeue_transmit_locked(task);  in xprt_request_dequeue_transmit()
1438  * xprt_request_dequeue_xprt - remove a task from the transmit+receive queue
1439  * @task: pointer to rpc_task
1441  * Remove a task from the transmit and receive queues, and ensure that
1445 xprt_request_dequeue_xprt(struct rpc_task *task)  in xprt_request_dequeue_xprt()  argument
1447 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_request_dequeue_xprt()
1450 	if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate) ||  in xprt_request_dequeue_xprt()
1451 	    test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) ||  in xprt_request_dequeue_xprt()
1455 			set_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate);  in xprt_request_dequeue_xprt()
1459 			clear_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate);  in xprt_request_dequeue_xprt()
1461 		xprt_request_dequeue_transmit_locked(task);  in xprt_request_dequeue_xprt()
1462 		xprt_request_dequeue_receive_locked(task);  in xprt_request_dequeue_xprt()
1488  * xprt_request_need_retransmit - Test if a task needs retransmission
1489  * @task: pointer to rpc_task
1491  * Test for whether a connection breakage requires the task to retransmit
1494 xprt_request_need_retransmit(struct rpc_task *task)  in xprt_request_need_retransmit()  argument
1496 	return xprt_request_retransmit_after_disconnect(task);  in xprt_request_need_retransmit()
1501  * @task: RPC task about to send a request
1504 bool xprt_prepare_transmit(struct rpc_task *task)  in xprt_prepare_transmit()  argument
1506 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_prepare_transmit()
1509 	if (!xprt_lock_write(xprt, task)) {  in xprt_prepare_transmit()
1511 		if (!test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_prepare_transmit()
1513 					task, 0);  in xprt_prepare_transmit()
1523 void xprt_end_transmit(struct rpc_task *task)  in xprt_end_transmit()  argument
1525 	struct rpc_xprt	*xprt = task->tk_rqstp->rq_xprt;  in xprt_end_transmit()
1528 	xprt_release_write(xprt, task);  in xprt_end_transmit()
1534  * @snd_task: RPC task that owns the transport lock
1545 	struct rpc_task *task = req->rq_task;  in xprt_request_transmit()  local
1547 	int is_retrans = RPC_WAS_SENT(task);  in xprt_request_transmit()
1554 		if (xprt_request_data_received(task)) {  in xprt_request_transmit()
1559 		if (rpcauth_xmit_need_reencode(task)) {  in xprt_request_transmit()
1563 		if (RPC_SIGNALLED(task)) {  in xprt_request_transmit()
1576 	trace_rpc_xdr_sendto(task, &req->rq_snd_buf);  in xprt_request_transmit()
1586 		task->tk_client->cl_stats->rpcretrans++;  in xprt_request_transmit()
1592 	task->tk_flags |= RPC_TASK_SENT;  in xprt_request_transmit()
1605 	xprt_request_dequeue_transmit(task);  in xprt_request_transmit()
1606 	rpc_wake_up_queued_task_set_status(&xprt->sending, task, status);  in xprt_request_transmit()
1612  * @task: controlling RPC task
1616  * resume, or @task finished transmitting, and detected that it already
1620 xprt_transmit(struct rpc_task *task)  in xprt_transmit()  argument
1622 	struct rpc_rqst *next, *req = task->tk_rqstp;  in xprt_transmit()
1634 		status = xprt_request_transmit(next, task);  in xprt_transmit()
1640 			if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_transmit()
1641 				task->tk_status = status;  in xprt_transmit()
1644 		/* Was @task transmitted, and has it received a reply? */  in xprt_transmit()
1645 		if (xprt_request_data_received(task) &&  in xprt_transmit()
1646 		    !test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_transmit()
1653 static void xprt_complete_request_init(struct rpc_task *task)  in xprt_complete_request_init()  argument
1655 	if (task->tk_rqstp)  in xprt_complete_request_init()
1656 		xprt_request_init(task);  in xprt_complete_request_init()
1659 void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_add_backlog()  argument
1662 	rpc_sleep_on(&xprt->backlog, task, xprt_complete_request_init);  in xprt_add_backlog()
1666 static bool __xprt_set_rq(struct rpc_task *task, void *data)  in __xprt_set_rq()  argument
1670 	if (task->tk_rqstp == NULL) {  in __xprt_set_rq()
1672 		task->tk_rqstp = req;  in __xprt_set_rq()
1688 static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_throttle_congested()  argument
1696 		xprt_add_backlog(xprt, task);  in xprt_throttle_congested()
1732 void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_alloc_slot()  argument
1749 		task->tk_status = -ENOMEM;  in xprt_alloc_slot()
1752 		xprt_add_backlog(xprt, task);  in xprt_alloc_slot()
1756 		task->tk_status = -EAGAIN;  in xprt_alloc_slot()
1765 	task->tk_status = 0;  in xprt_alloc_slot()
1766 	task->tk_rqstp = req;  in xprt_alloc_slot()
1884 xprt_request_init(struct rpc_task *task)  in xprt_request_init()  argument
1886 	struct rpc_xprt *xprt = task->tk_xprt;  in xprt_request_init()
1887 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_request_init()
1889 	req->rq_task	= task;  in xprt_request_init()
1901 	xprt_init_majortimeo(task, req, task->tk_client->cl_timeout);  in xprt_request_init()
1907 xprt_do_reserve(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_do_reserve()  argument
1909 	xprt->ops->alloc_slot(xprt, task);  in xprt_do_reserve()
1910 	if (task->tk_rqstp != NULL)  in xprt_do_reserve()
1911 		xprt_request_init(task);  in xprt_do_reserve()
1916  * @task: RPC task requesting a slot allocation
1919  * slots are available, place the task on the transport's
1922 void xprt_reserve(struct rpc_task *task)  in xprt_reserve()  argument
1924 	struct rpc_xprt *xprt = task->tk_xprt;  in xprt_reserve()
1926 	task->tk_status = 0;  in xprt_reserve()
1927 	if (task->tk_rqstp != NULL)  in xprt_reserve()
1930 	task->tk_status = -EAGAIN;  in xprt_reserve()
1931 	if (!xprt_throttle_congested(xprt, task))  in xprt_reserve()
1932 		xprt_do_reserve(xprt, task);  in xprt_reserve()
1937  * @task: RPC task requesting a slot allocation
1939  * If no more slots are available, place the task on the transport's
1944 void xprt_retry_reserve(struct rpc_task *task)  in xprt_retry_reserve()  argument
1946 	struct rpc_xprt *xprt = task->tk_xprt;  in xprt_retry_reserve()
1948 	task->tk_status = 0;  in xprt_retry_reserve()
1949 	if (task->tk_rqstp != NULL)  in xprt_retry_reserve()
1952 	task->tk_status = -EAGAIN;  in xprt_retry_reserve()
1953 	xprt_do_reserve(xprt, task);  in xprt_retry_reserve()
1958  * @task: task which is finished with the slot
1961 void xprt_release(struct rpc_task *task)  in xprt_release()  argument
1964 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_release()
1967 		if (task->tk_client) {  in xprt_release()
1968 			xprt = task->tk_xprt;  in xprt_release()
1969 			xprt_release_write(xprt, task);  in xprt_release()
1975 	xprt_request_dequeue_xprt(task);  in xprt_release()
1977 	xprt->ops->release_xprt(xprt, task);  in xprt_release()
1979 		xprt->ops->release_request(task);  in xprt_release()
1983 		xprt->ops->buf_free(task);  in xprt_release()
1989 	task->tk_rqstp = NULL;  in xprt_release()
1998 xprt_init_bc_request(struct rpc_rqst *req, struct rpc_task *task,  in xprt_init_bc_request()  argument
2003 	task->tk_rqstp = req;  in xprt_init_bc_request()
2004 	req->rq_task = task;  in xprt_init_bc_request()
2018 	xprt_init_majortimeo(task, req, to);  in xprt_init_bc_request()