23 bool rxrpc_propose_abort(struct rxrpc_call *call, s32 abort_code, int error,  in rxrpc_propose_abort()  argument
26 	_enter("{%d},%d,%d,%u", call->debug_id, abort_code, error, why);  in rxrpc_propose_abort()
28 	if (!call->send_abort && !rxrpc_call_is_complete(call)) {  in rxrpc_propose_abort()
29 		call->send_abort_why = why;  in rxrpc_propose_abort()
30 		call->send_abort_err = error;  in rxrpc_propose_abort()
31 		call->send_abort_seq = 0;  in rxrpc_propose_abort()
32 		trace_rxrpc_abort_call(call, abort_code);  in rxrpc_propose_abort()
34 		smp_store_release(&call->send_abort, abort_code);  in rxrpc_propose_abort()
35 		rxrpc_poke_call(call, rxrpc_call_poke_abort);  in rxrpc_propose_abort()
43  * Wait for a call to become connected.  Interruption here doesn't cause the
44  * call to be aborted.
46 static int rxrpc_wait_to_be_connected(struct rxrpc_call *call, long *timeo)  in rxrpc_wait_to_be_connected()  argument
51 	_enter("%d", call->debug_id);  in rxrpc_wait_to_be_connected()
53 	if (rxrpc_call_state(call) != RXRPC_CALL_CLIENT_AWAIT_CONN)  in rxrpc_wait_to_be_connected()
56 	add_wait_queue_exclusive(&call->waitq, &myself);  in rxrpc_wait_to_be_connected()
59 		switch (call->interruptibility) {  in rxrpc_wait_to_be_connected()
70 		if (rxrpc_call_state(call) != RXRPC_CALL_CLIENT_AWAIT_CONN)  in rxrpc_wait_to_be_connected()
72 		if ((call->interruptibility == RXRPC_INTERRUPTIBLE ||  in rxrpc_wait_to_be_connected()
73 		     call->interruptibility == RXRPC_PREINTERRUPTIBLE) &&  in rxrpc_wait_to_be_connected()
81 	remove_wait_queue(&call->waitq, &myself);  in rxrpc_wait_to_be_connected()
85 	if (ret == 0 && rxrpc_call_is_complete(call))  in rxrpc_wait_to_be_connected()
86 		ret = call->error;  in rxrpc_wait_to_be_connected()
95 static bool rxrpc_check_tx_space(struct rxrpc_call *call, rxrpc_seq_t *_tx_win)  in rxrpc_check_tx_space()  argument
97 	rxrpc_seq_t tx_bottom = READ_ONCE(call->tx_bottom);  in rxrpc_check_tx_space()
101 	return call->send_top - tx_bottom < 256;  in rxrpc_check_tx_space()
108 					 struct rxrpc_call *call,  in rxrpc_wait_for_tx_window_intr()  argument
113 		if (rxrpc_check_tx_space(call, NULL))  in rxrpc_wait_for_tx_window_intr()
116 		if (rxrpc_call_is_complete(call))  in rxrpc_wait_for_tx_window_intr()
117 			return call->error;  in rxrpc_wait_for_tx_window_intr()
122 		trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);  in rxrpc_wait_for_tx_window_intr()
132 					    struct rxrpc_call *call)  in rxrpc_wait_for_tx_window_waitall()  argument
137 	rtt = READ_ONCE(call->srtt_us) >> 3;  in rxrpc_wait_for_tx_window_waitall()
143 	tx_start = READ_ONCE(call->tx_bottom);  in rxrpc_wait_for_tx_window_waitall()
148 		if (rxrpc_check_tx_space(call, &tx_win))  in rxrpc_wait_for_tx_window_waitall()
151 		if (rxrpc_call_is_complete(call))  in rxrpc_wait_for_tx_window_waitall()
152 			return call->error;  in rxrpc_wait_for_tx_window_waitall()
163 		trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);  in rxrpc_wait_for_tx_window_waitall()
172 					    struct rxrpc_call *call,  in rxrpc_wait_for_tx_window_nonintr()  argument
177 		if (rxrpc_check_tx_space(call, NULL))  in rxrpc_wait_for_tx_window_nonintr()
180 		if (rxrpc_call_is_complete(call))  in rxrpc_wait_for_tx_window_nonintr()
181 			return call->error;  in rxrpc_wait_for_tx_window_nonintr()
183 		trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);  in rxrpc_wait_for_tx_window_nonintr()
193 				    struct rxrpc_call *call,  in rxrpc_wait_for_tx_window()  argument
201 	       call->tx_bottom, call->tx_top, call->tx_winsize);  in rxrpc_wait_for_tx_window()
203 	add_wait_queue(&call->waitq, &myself);  in rxrpc_wait_for_tx_window()
205 	switch (call->interruptibility) {  in rxrpc_wait_for_tx_window()
208 			ret = rxrpc_wait_for_tx_window_waitall(rx, call);  in rxrpc_wait_for_tx_window()
210 			ret = rxrpc_wait_for_tx_window_intr(rx, call, timeo);  in rxrpc_wait_for_tx_window()
215 		ret = rxrpc_wait_for_tx_window_nonintr(rx, call, timeo);  in rxrpc_wait_for_tx_window()
219 	remove_wait_queue(&call->waitq, &myself);  in rxrpc_wait_for_tx_window()
226  * Notify the owner of the call that the transmit phase is ended and the last
229 static void rxrpc_notify_end_tx(struct rxrpc_sock *rx, struct rxrpc_call *call,  in rxrpc_notify_end_tx()  argument
233 		notify_end_tx(&rx->sk, call, call->user_call_ID);  in rxrpc_notify_end_tx()
241 static void rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call,  in rxrpc_queue_packet()  argument
245 	struct rxrpc_txqueue *sq = call->send_queue;  in rxrpc_queue_packet()
249 	rxrpc_inc_stat(call->rxnet, stat_tx_data);  in rxrpc_queue_packet()
251 	ASSERTCMP(txb->seq, ==, call->send_top + 1);  in rxrpc_queue_packet()
254 		trace_rxrpc_txqueue(call, rxrpc_txqueue_queue_last);  in rxrpc_queue_packet()
256 		trace_rxrpc_txqueue(call, rxrpc_txqueue_queue);  in rxrpc_queue_packet()
259 		trace_rxrpc_tq(call, sq, seq, rxrpc_tq_queue_dup);  in rxrpc_queue_packet()
261 		trace_rxrpc_tq(call, sq, seq, rxrpc_tq_queue);  in rxrpc_queue_packet()
263 	/* Add the packet to the call's output buffer */  in rxrpc_queue_packet()
264 	poke = (READ_ONCE(call->tx_bottom) == call->send_top);  in rxrpc_queue_packet()
267 	smp_store_release(&call->send_top, seq);  in rxrpc_queue_packet()
269 		set_bit(RXRPC_CALL_TX_NO_MORE, &call->flags);  in rxrpc_queue_packet()
270 		rxrpc_notify_end_tx(rx, call, notify_end_tx);  in rxrpc_queue_packet()
271 		call->send_queue = NULL;  in rxrpc_queue_packet()
275 		rxrpc_poke_call(call, rxrpc_call_poke_start);  in rxrpc_queue_packet()
281 static int rxrpc_alloc_txqueue(struct sock *sk, struct rxrpc_call *call)  in rxrpc_alloc_txqueue()  argument
293 	if (call->send_queue) {  in rxrpc_alloc_txqueue()
294 		tq->qbase = call->send_top + 1;  in rxrpc_alloc_txqueue()
295 		call->send_queue->next = tq;  in rxrpc_alloc_txqueue()
296 		call->send_queue = tq;  in rxrpc_alloc_txqueue()
297 	} else if (WARN_ON(call->tx_queue)) {  in rxrpc_alloc_txqueue()
305 		call->tx_qbase = 0;  in rxrpc_alloc_txqueue()
306 		call->send_queue = tq;  in rxrpc_alloc_txqueue()
307 		call->tx_qtail = tq;  in rxrpc_alloc_txqueue()
308 		call->tx_queue = tq;  in rxrpc_alloc_txqueue()
311 	trace_rxrpc_tq(call, tq, call->send_top, rxrpc_tq_alloc);  in rxrpc_alloc_txqueue()
318  * - The caller holds the call user access mutex, but not the socket lock.
321 			   struct rxrpc_call *call,  in rxrpc_send_data()  argument
333 	if (test_bit(RXRPC_CALL_TX_NO_MORE, &call->flags)) {  in rxrpc_send_data()
334 		trace_rxrpc_abort(call->debug_id, rxrpc_sendmsg_late_send,  in rxrpc_send_data()
335 				  call->cid, call->call_id, call->rx_consumed,  in rxrpc_send_data()
342 	ret = rxrpc_wait_to_be_connected(call, &timeo);  in rxrpc_send_data()
346 	if (call->conn->state == RXRPC_CONN_CLIENT_UNSECURED) {  in rxrpc_send_data()
347 		ret = rxrpc_init_client_conn_security(call->conn);  in rxrpc_send_data()
356 	txb = call->tx_pending;  in rxrpc_send_data()
357 	call->tx_pending = NULL;  in rxrpc_send_data()
364 	state = rxrpc_call_state(call);  in rxrpc_send_data()
372 		/* Request phase complete for this client call */  in rxrpc_send_data()
373 		trace_rxrpc_abort(call->debug_id, rxrpc_sendmsg_late_send,  in rxrpc_send_data()
374 				  call->cid, call->call_id, call->rx_consumed,  in rxrpc_send_data()
380 	if (call->tx_total_len != -1) {  in rxrpc_send_data()
381 		if (len - copied > call->tx_total_len)  in rxrpc_send_data()
383 		if (!more && len - copied != call->tx_total_len)  in rxrpc_send_data()
393 			if (!rxrpc_check_tx_space(call, NULL))  in rxrpc_send_data()
397 			if (!call->send_queue || !((call->send_top + 1) & RXRPC_TXQ_MASK)) {  in rxrpc_send_data()
398 				ret = rxrpc_alloc_txqueue(sk, call);  in rxrpc_send_data()
408 			txb = call->conn->security->alloc_txbuf(call, remain, sk->sk_allocation);  in rxrpc_send_data()
430 			if (call->tx_total_len != -1)  in rxrpc_send_data()
431 				call->tx_total_len -= copy;  in rxrpc_send_data()
434 		/* check for the far side aborting the call or a network error  in rxrpc_send_data()
436 		if (rxrpc_call_is_complete(call))  in rxrpc_send_data()
445 			ret = call->security->secure_packet(call, txb);  in rxrpc_send_data()
448 			rxrpc_queue_packet(rx, call, txb, notify_end_tx);  in rxrpc_send_data()
455 	if (rxrpc_call_is_complete(call) &&  in rxrpc_send_data()
456 	    call->error < 0)  in rxrpc_send_data()
457 		ret = call->error;  in rxrpc_send_data()
459 	call->tx_pending = txb;  in rxrpc_send_data()
465 	_leave(" = %d", call->error);  in rxrpc_send_data()
466 	return call->error;  in rxrpc_send_data()
481 	mutex_unlock(&call->user_mutex);  in rxrpc_send_data()
483 	ret = rxrpc_wait_for_tx_window(rx, call, &timeo,  in rxrpc_send_data()
487 	if (call->interruptibility == RXRPC_INTERRUPTIBLE) {  in rxrpc_send_data()
488 		if (mutex_lock_interruptible(&call->user_mutex) < 0) {  in rxrpc_send_data()
493 		mutex_lock(&call->user_mutex);  in rxrpc_send_data()
527 				p->call.user_call_ID = *(u32 *)CMSG_DATA(cmsg);  in rxrpc_sendmsg_cmsg()
531 				p->call.user_call_ID = *(unsigned long *)  in rxrpc_sendmsg_cmsg()
569 			if (p->call.tx_total_len != -1 || len != sizeof(__s64))  in rxrpc_sendmsg_cmsg()
571 			p->call.tx_total_len = *(__s64 *)CMSG_DATA(cmsg);  in rxrpc_sendmsg_cmsg()
572 			if (p->call.tx_total_len < 0)  in rxrpc_sendmsg_cmsg()
579 			memcpy(&p->call.timeouts, CMSG_DATA(cmsg), len);  in rxrpc_sendmsg_cmsg()
580 			p->call.nr_timeouts = len / 4;  in rxrpc_sendmsg_cmsg()
581 			if (p->call.timeouts.hard > INT_MAX / HZ)  in rxrpc_sendmsg_cmsg()
583 			if (p->call.nr_timeouts >= 2 && p->call.timeouts.idle > 60 * 60 * 1000)  in rxrpc_sendmsg_cmsg()
585 			if (p->call.nr_timeouts >= 3 && p->call.timeouts.normal > 60 * 60 * 1000)  in rxrpc_sendmsg_cmsg()
596 	if (p->call.tx_total_len != -1 && p->command != RXRPC_CMD_SEND_DATA)  in rxrpc_sendmsg_cmsg()
603  * Create a new client call for sendmsg().
605  * - If it returns a call, the call's lock will need releasing by the caller.
611 	__acquires(&call->user_mutex)  in rxrpc_new_client_call_for_sendmsg()
615 	struct rxrpc_call *call;  in rxrpc_new_client_call_for_sendmsg()  local
645 	call = rxrpc_new_client_call(rx, &cp, &p->call, GFP_KERNEL,  in rxrpc_new_client_call_for_sendmsg()
650 	_leave(" = %p\n", call);  in rxrpc_new_client_call_for_sendmsg()
651 	return call;  in rxrpc_new_client_call_for_sendmsg()
655  * send a message forming part of a client call through an RxRPC socket
662 	struct rxrpc_call *call;  in rxrpc_do_sendmsg()  local
667 		.call.tx_total_len	= -1,  in rxrpc_do_sendmsg()
668 		.call.user_call_ID	= 0,  in rxrpc_do_sendmsg()
669 		.call.nr_timeouts	= 0,  in rxrpc_do_sendmsg()
670 		.call.interruptibility	= RXRPC_INTERRUPTIBLE,  in rxrpc_do_sendmsg()
687 		ret = rxrpc_user_charge_accept(rx, p.call.user_call_ID);  in rxrpc_do_sendmsg()
691 	call = rxrpc_find_call_by_user_ID(rx, p.call.user_call_ID);  in rxrpc_do_sendmsg()
692 	if (!call) {  in rxrpc_do_sendmsg()
696 		call = rxrpc_new_client_call_for_sendmsg(rx, msg, &p);  in rxrpc_do_sendmsg()
698 		if (IS_ERR(call))  in rxrpc_do_sendmsg()
699 			return PTR_ERR(call);  in rxrpc_do_sendmsg()
700 		/* ... and we have the call lock. */  in rxrpc_do_sendmsg()
701 		p.call.nr_timeouts = 0;  in rxrpc_do_sendmsg()
703 		if (rxrpc_call_is_complete(call))  in rxrpc_do_sendmsg()
706 		switch (rxrpc_call_state(call)) {  in rxrpc_do_sendmsg()
714 			rxrpc_put_call(call, rxrpc_call_put_sendmsg);  in rxrpc_do_sendmsg()
721 		ret = mutex_lock_interruptible(&call->user_mutex);  in rxrpc_do_sendmsg()
728 		if (p.call.tx_total_len != -1) {  in rxrpc_do_sendmsg()
730 			if (call->tx_total_len != -1 ||  in rxrpc_do_sendmsg()
731 			    call->tx_pending ||  in rxrpc_do_sendmsg()
732 			    call->tx_top != 0)  in rxrpc_do_sendmsg()
734 			call->tx_total_len = p.call.tx_total_len;  in rxrpc_do_sendmsg()
738 	switch (p.call.nr_timeouts) {  in rxrpc_do_sendmsg()
740 		WRITE_ONCE(call->next_rx_timo, p.call.timeouts.normal);  in rxrpc_do_sendmsg()
743 		WRITE_ONCE(call->next_req_timo, p.call.timeouts.idle);  in rxrpc_do_sendmsg()
746 		if (p.call.timeouts.hard > 0) {  in rxrpc_do_sendmsg()
747 			ktime_t delay = ms_to_ktime(p.call.timeouts.hard * MSEC_PER_SEC);  in rxrpc_do_sendmsg()
749 			WRITE_ONCE(call->expect_term_by,  in rxrpc_do_sendmsg()
750 				   ktime_add(p.call.timeouts.hard,  in rxrpc_do_sendmsg()
752 			trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_hard);  in rxrpc_do_sendmsg()
753 			rxrpc_poke_call(call, rxrpc_call_poke_set_timeout);  in rxrpc_do_sendmsg()
759 	if (rxrpc_call_is_complete(call)) {  in rxrpc_do_sendmsg()
760 		/* it's too late for this call */  in rxrpc_do_sendmsg()
763 		rxrpc_propose_abort(call, p.abort_code, -ECONNABORTED,  in rxrpc_do_sendmsg()
769 		ret = rxrpc_send_data(rx, call, msg, len, NULL, &dropped_lock);  in rxrpc_do_sendmsg()
774 		mutex_unlock(&call->user_mutex);  in rxrpc_do_sendmsg()
776 	rxrpc_put_call(call, rxrpc_call_put_sendmsg);  in rxrpc_do_sendmsg()
786  * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
787  * @sock: The socket the call is on
788  * @call: The call to send data through
793  * Allow a kernel service to send data on a call.  The call must be in an state
798 int rxrpc_kernel_send_data(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_send_data()  argument
805 	_enter("{%d},", call->debug_id);  in rxrpc_kernel_send_data()
810 	mutex_lock(&call->user_mutex);  in rxrpc_kernel_send_data()
812 	ret = rxrpc_send_data(rxrpc_sk(sock->sk), call, msg, len,  in rxrpc_kernel_send_data()
815 		ret = call->error;  in rxrpc_kernel_send_data()
818 		mutex_unlock(&call->user_mutex);  in rxrpc_kernel_send_data()
825  * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
826  * @sock: The socket the call is on
827  * @call: The call to be aborted
832  * Allow a kernel service to abort a call, if it's still in an abortable state
833  * and return true if the call was aborted, false if it was already complete.
835 bool rxrpc_kernel_abort_call(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_abort_call()  argument
840 	_enter("{%d},%d,%d,%u", call->debug_id, abort_code, error, why);  in rxrpc_kernel_abort_call()
842 	mutex_lock(&call->user_mutex);  in rxrpc_kernel_abort_call()
843 	aborted = rxrpc_propose_abort(call, abort_code, error, why);  in rxrpc_kernel_abort_call()
844 	mutex_unlock(&call->user_mutex);  in rxrpc_kernel_abort_call()
850  * rxrpc_kernel_set_tx_length - Set the total Tx length on a call
851  * @sock: The socket the call is on
852  * @call: The call to be informed
853  * @tx_total_len: The amount of data to be transmitted for this call
855  * Allow a kernel service to set the total transmit length on a call.  This
859  * request length can be set when beginning the call.
861 void rxrpc_kernel_set_tx_length(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_set_tx_length()  argument
864 	WARN_ON(call->tx_total_len != -1);  in rxrpc_kernel_set_tx_length()
865 	call->tx_total_len = tx_total_len;  in rxrpc_kernel_set_tx_length()