70 	struct mptcp_subflow_context *subflow;  in __mptcp_socket_create()  local
81 	subflow = mptcp_subflow_ctx(ssock->sk);  in __mptcp_socket_create()
82 	list_add(&subflow->node, &msk->conn_list);  in __mptcp_socket_create()
84 	subflow->request_mptcp = 1;  in __mptcp_socket_create()
85 	subflow->subflow_id = msk->subflow_id++;  in __mptcp_socket_create()
87 	/* This is the first subflow, always with id 0 */  in __mptcp_socket_create()
88 	WRITE_ONCE(subflow->local_id, 0);  in __mptcp_socket_create()
95 /* If the MPC handshake is not started, returns the first subflow,
343 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_move_skb()  local
353 	/* try to fetch required memory from subflow */  in __mptcp_move_skb()
365 	MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow);  in __mptcp_move_skb()
485 static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow)  in mptcp_timeout_from_subflow()  argument
487 	const struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_timeout_from_subflow()
489 	return inet_csk(ssk)->icsk_pending && !subflow->stale_count ?  in mptcp_timeout_from_subflow()
495 	struct mptcp_subflow_context *subflow;  in mptcp_set_timeout()  local
498 	mptcp_for_each_subflow(mptcp_sk(sk), subflow)  in mptcp_set_timeout()
499 		tout = max(tout, mptcp_timeout_from_subflow(subflow));  in mptcp_set_timeout()
526 	struct mptcp_subflow_context *subflow;  in mptcp_send_ack()  local
528 	mptcp_for_each_subflow(msk, subflow)  in mptcp_send_ack()
529 		mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow));  in mptcp_send_ack()
558 	struct mptcp_subflow_context *subflow;  in mptcp_cleanup_rbuf()  local
566 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_cleanup_rbuf()
567 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_cleanup_rbuf()
583 	 * at the subflow level and the msk lock was not held, so this  in mptcp_check_data_fin()
640 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_move_skbs_from_subflow()  local
668 		map_remaining = subflow->map_data_len -  in __mptcp_move_skbs_from_subflow()
669 				mptcp_subflow_get_map_offset(subflow);  in __mptcp_move_skbs_from_subflow()
688 			subflow->map_data_len = skb->len;  in __mptcp_move_skbs_from_subflow()
814 	struct mptcp_subflow_context *subflow;  in __mptcp_error_report()  local
817 	mptcp_for_each_subflow(msk, subflow)  in __mptcp_error_report()
818 		if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow)))  in __mptcp_error_report()
851 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_data_ready()  local
856 	 * subflow at msk destruction time, but we must avoid enqueuing  in mptcp_data_ready()
859 	if (unlikely(subflow->disposable))  in mptcp_data_ready()
908 	struct mptcp_subflow_context *tmp, *subflow;  in __mptcp_flush_join_list()  local
911 	list_for_each_entry_safe(subflow, tmp, join_list, node) {  in __mptcp_flush_join_list()
912 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_flush_join_list()
915 		list_move_tail(&subflow->node, &msk->conn_list);  in __mptcp_flush_join_list()
944 		/* each subflow already holds a reference to the sk, and the  in mptcp_schedule_work()
945 		 * workqueue is invoked by a subflow, so sk can't go away here.  in mptcp_schedule_work()
955 	struct mptcp_subflow_context *subflow;  in mptcp_subflow_recv_lookup()  local
959 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_subflow_recv_lookup()
960 		if (READ_ONCE(subflow->data_avail))  in mptcp_subflow_recv_lookup()
961 			return mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_recv_lookup()
1088 	struct mptcp_subflow_context *subflow;  in mptcp_enter_memory_pressure()  local
1092 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_enter_memory_pressure()
1093 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_enter_memory_pressure()
1396 void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow)  in mptcp_subflow_set_active()  argument
1398 	if (!subflow->stale)  in mptcp_subflow_set_active()
1401 	subflow->stale = 0;  in mptcp_subflow_set_active()
1402 	MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER);  in mptcp_subflow_set_active()
1405 bool mptcp_subflow_active(struct mptcp_subflow_context *subflow)  in mptcp_subflow_active()  argument
1407 	if (unlikely(subflow->stale)) {  in mptcp_subflow_active()
1408 		u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp);  in mptcp_subflow_active()
1410 		if (subflow->stale_rcv_tstamp == rcv_tstamp)  in mptcp_subflow_active()
1413 		mptcp_subflow_set_active(subflow);  in mptcp_subflow_active()
1415 	return __mptcp_subflow_active(subflow);  in mptcp_subflow_active()
1423  * returns the subflow that will transmit the next DSS
1429 	struct mptcp_subflow_context *subflow;  in mptcp_subflow_get_send()  local
1437 	/* pick the subflow with the lower wmem/wspace ratio */  in mptcp_subflow_get_send()
1443 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_subflow_get_send()
1444 		bool backup = subflow->backup || subflow->request_bkup;  in mptcp_subflow_get_send()
1446 		trace_mptcp_subflow_get_send(subflow);  in mptcp_subflow_get_send()
1447 		ssk =  mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_send()
1448 		if (!mptcp_subflow_active(subflow))  in mptcp_subflow_get_send()
1451 		tout = max(tout, mptcp_timeout_from_subflow(subflow));  in mptcp_subflow_get_send()
1453 		pace = subflow->avg_pacing_rate;  in mptcp_subflow_get_send()
1456 			subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate);  in mptcp_subflow_get_send()
1457 			pace = subflow->avg_pacing_rate;  in mptcp_subflow_get_send()
1470 	/* pick the best backup if no other subflow is active */  in mptcp_subflow_get_send()
1480 	 *   otherwise do not use the picked, slower, subflow  in mptcp_subflow_get_send()
1481 	 * We select the subflow with the shorter estimated time to flush  in mptcp_subflow_get_send()
1483 	 * to check that subflow has a non empty cwin.  in mptcp_subflow_get_send()
1494 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_get_send()
1495 	subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem +  in mptcp_subflow_get_send()
1526 	 * that has been handed to the subflow for transmission  in mptcp_update_post_push()
1599 		struct mptcp_subflow_context *subflow;  in __mptcp_push_pending()  local
1607 		mptcp_for_each_subflow(msk, subflow) {  in __mptcp_push_pending()
1608 			if (READ_ONCE(subflow->scheduled)) {  in __mptcp_push_pending()
1609 				mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_push_pending()
1612 				ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_push_pending()
1620 					/* Need to lock the new subflow only if different  in __mptcp_push_pending()
1642 	/* at this point we held the socket lock for the last subflow we used */  in __mptcp_push_pending()
1665 		struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_subflow_push_pending()  local
1668 		/* check for a different subflow usage only after  in __mptcp_subflow_push_pending()
1672 			mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_subflow_push_pending()
1684 		if (READ_ONCE(subflow->scheduled)) {  in __mptcp_subflow_push_pending()
1685 			mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_subflow_push_pending()
1692 		mptcp_for_each_subflow(msk, subflow) {  in __mptcp_subflow_push_pending()
1693 			if (READ_ONCE(subflow->scheduled)) {  in __mptcp_subflow_push_pending()
1694 				xmit_ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_subflow_push_pending()
1696 					mptcp_subflow_delegate(subflow,  in __mptcp_subflow_push_pending()
1731 	 * first subflow right now. Otherwise we are in the defer_connect  in mptcp_sendmsg_fastopen()
1732 	 * path, and the first subflow must be already present.  in mptcp_sendmsg_fastopen()
1734 	 * fastopen attempt, no need to check for additional subflow status.  in mptcp_sendmsg_fastopen()
2010 	struct mptcp_subflow_context *subflow;  in mptcp_rcv_space_adjust()  local
2034 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_rcv_space_adjust()
2039 		tp = tcp_sk(mptcp_subflow_tcp_sock(subflow));  in mptcp_rcv_space_adjust()
2079 			 * get drops at subflow level if skbs can't be moved to  in mptcp_rcv_space_adjust()
2083 			mptcp_for_each_subflow(msk, subflow) {  in mptcp_rcv_space_adjust()
2087 				ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_rcv_space_adjust()
2332 /* Find an idle subflow.  Return NULL if there is unacked data at tcp
2335  * A backup subflow is returned only if that is the only kind available.
2340 	struct mptcp_subflow_context *subflow;  in mptcp_subflow_get_retrans()  local
2343 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_subflow_get_retrans()
2344 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_retrans()
2346 		if (!__mptcp_subflow_active(subflow))  in mptcp_subflow_get_retrans()
2352 			min_stale_count = min_t(int, min_stale_count, subflow->stale_count);  in mptcp_subflow_get_retrans()
2356 		if (subflow->backup || subflow->request_bkup) {  in mptcp_subflow_get_retrans()
2415  * clean completely the subflow status when the subflow reaches
2419 				       struct mptcp_subflow_context *subflow,  in __mptcp_subflow_disconnect()  argument
2424 		/* The MPTCP code never wait on the subflow sockets, TCP-level  in __mptcp_subflow_disconnect()
2428 		mptcp_subflow_ctx_reset(subflow);  in __mptcp_subflow_disconnect()
2434 /* subflow sockets can be either outgoing (connect) or incoming
2443 			      struct mptcp_subflow_context *subflow,  in __mptcp_close_ssk()  argument
2449 	/* If the first subflow moved to a close state before accept, e.g. due  in __mptcp_close_ssk()
2450 	 * to an incoming reset or listener shutdown, the subflow socket is  in __mptcp_close_ssk()
2466 		list_del(&subflow->node);  in __mptcp_close_ssk()
2476 		subflow->send_fastclose = 1;  in __mptcp_close_ssk()
2481 		__mptcp_subflow_disconnect(ssk, subflow, flags);  in __mptcp_close_ssk()
2487 	subflow->disposable = 1;  in __mptcp_close_ssk()
2495 		kfree_rcu(subflow, rcu);  in __mptcp_close_ssk()
2497 		/* otherwise tcp will dispose of the ssk and subflow ctx */  in __mptcp_close_ssk()
2536 		     struct mptcp_subflow_context *subflow)  in mptcp_close_ssk()  argument
2538 	/* The first subflow can already be closed and still in the list */  in mptcp_close_ssk()
2539 	if (subflow->close_event_done)  in mptcp_close_ssk()
2542 	subflow->close_event_done = true;  in mptcp_close_ssk()
2547 	/* subflow aborted before reaching the fully_established status  in mptcp_close_ssk()
2548 	 * attempt the creation of the next subflow  in mptcp_close_ssk()
2550 	mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow);  in mptcp_close_ssk()
2552 	__mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH);  in mptcp_close_ssk()
2562 	struct mptcp_subflow_context *subflow, *tmp;  in __mptcp_close_subflow()  local
2567 	mptcp_for_each_subflow_safe(msk, subflow, tmp) {  in __mptcp_close_subflow()
2568 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_close_subflow()
2580 		mptcp_close_ssk(sk, ssk, subflow);  in __mptcp_close_subflow()
2597 	struct mptcp_subflow_context *subflow, *tmp;  in mptcp_check_fastclose()  local
2605 	mptcp_for_each_subflow_safe(msk, subflow, tmp) {  in mptcp_check_fastclose()
2606 		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);  in mptcp_check_fastclose()
2647 	struct mptcp_subflow_context *subflow;  in __mptcp_retrans()  local
2679 	mptcp_for_each_subflow(msk, subflow) {  in __mptcp_retrans()
2680 		if (READ_ONCE(subflow->scheduled)) {  in __mptcp_retrans()
2683 			mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_retrans()
2685 			ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_retrans()
2753 	pr_debug("MP_FAIL doesn't respond, reset the subflow\n");  in mptcp_mp_fail_no_response()
2763 	struct mptcp_subflow_context *subflow, *tmp;  in mptcp_do_fastclose()  local
2767 	mptcp_for_each_subflow_safe(msk, subflow, tmp)  in mptcp_do_fastclose()
2768 		__mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow),  in mptcp_do_fastclose()
2769 				  subflow, MPTCP_CF_FASTCLOSE);  in mptcp_do_fastclose()
2939 			pr_debug("Sending DATA_FIN on subflow %p\n", ssk);  in mptcp_subflow_shutdown()
3002 	struct mptcp_subflow_context *subflow;  in mptcp_check_send_data_fin()  local
3018 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_check_send_data_fin()
3019 		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);  in mptcp_check_send_data_fin()
3097 	struct mptcp_subflow_context *subflow;  in __mptcp_close()  local
3124 	mptcp_for_each_subflow(msk, subflow) {  in __mptcp_close()
3125 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_close()
3134 			subflow->fail_tout = 0;  in __mptcp_close()
3221 	/* msk->subflow is still intact, the following will not free the first  in mptcp_disconnect()
3222 	 * subflow  in mptcp_disconnect()
3307 	struct mptcp_subflow_context *subflow;  in mptcp_sk_clone_init()  local
3342 	/* passive msk is created after the first/MPC subflow */  in mptcp_sk_clone_init()
3353 	/* The msk maintain a ref to each subflow in the connections list */  in mptcp_sk_clone_init()
3355 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_sk_clone_init()
3356 	list_add(&subflow->node, &msk->conn_list);  in mptcp_sk_clone_init()
3359 	/* new mpc subflow takes ownership of the newly  in mptcp_sk_clone_init()
3373 		__mptcp_subflow_fully_established(msk, subflow, mp_opt);  in mptcp_sk_clone_init()
3399 	struct mptcp_subflow_context *subflow, *tmp;  in mptcp_destroy_common()  local
3405 	mptcp_for_each_subflow_safe(msk, subflow, tmp)  in mptcp_destroy_common()
3406 		__mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags);  in mptcp_destroy_common()
3429 	/* allow the following to close even the initial subflow */  in mptcp_destroy()
3474 		/* the following actions acquire the subflow socket lock  in mptcp_release_cb()
3479 		 *    the subflow socket lock  in mptcp_release_cb()
3500 		 * On sk release avoid actions depending on the first subflow  in mptcp_release_cb()
3513 /* MP_JOIN client subflow must wait for 4th ack before sending any data:
3514  * TCP can't schedule delack timer before the subflow is fully established.
3542 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_process_delegated()  local
3543 	struct sock *sk = subflow->conn;  in mptcp_subflow_process_delegated()
3592 	struct mptcp_subflow_context *subflow;  in mptcp_finish_connect()  local
3596 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_finish_connect()
3597 	sk = subflow->conn;  in mptcp_finish_connect()
3600 	pr_debug("msk=%p, token=%u\n", sk, subflow->token);  in mptcp_finish_connect()
3602 	subflow->map_seq = subflow->iasn;  in mptcp_finish_connect()
3603 	subflow->map_subflow_seq = 1;  in mptcp_finish_connect()
3605 	/* the socket is not connected yet, no msk/subflow ops can access/race  in mptcp_finish_connect()
3608 	WRITE_ONCE(msk->local_key, subflow->local_key);  in mptcp_finish_connect()
3624 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_finish_join()  local
3625 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);  in mptcp_finish_join()
3629 	pr_debug("msk=%p, subflow=%p\n", msk, subflow);  in mptcp_finish_join()
3633 		subflow->reset_reason = MPTCP_RST_EMPTCP;  in mptcp_finish_join()
3637 	/* active subflow, already present inside the conn_list */  in mptcp_finish_join()
3638 	if (!list_empty(&subflow->node)) {  in mptcp_finish_join()
3655 			list_add_tail(&subflow->node, &msk->conn_list);  in mptcp_finish_join()
3659 		list_add_tail(&subflow->node, &msk->join_list);  in mptcp_finish_join()
3666 		subflow->reset_reason = MPTCP_RST_EPROHIBIT;  in mptcp_finish_join()
3702 		/* the first subflow is disconnected after close - see  in mptcp_ioctl_outq()
3750 	struct mptcp_subflow_context *subflow;  in mptcp_connect()  local
3760 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_connect()
3766 		mptcp_subflow_early_fallback(msk, subflow);  in mptcp_connect()
3768 	if (subflow->request_mptcp) {  in mptcp_connect()
3771 			mptcp_subflow_early_fallback(msk, subflow);  in mptcp_connect()
3774 			mptcp_subflow_early_fallback(msk, subflow);  in mptcp_connect()
3778 	WRITE_ONCE(msk->write_seq, subflow->idsn);  in mptcp_connect()
3779 	WRITE_ONCE(msk->snd_nxt, subflow->idsn);  in mptcp_connect()
3780 	WRITE_ONCE(msk->snd_una, subflow->idsn);  in mptcp_connect()
3785 	 * acquired the subflow socket lock, too.  in mptcp_connect()
3936 	 * but no need to allocate the first subflow just to error out.  in mptcp_stream_accept()
3947 	pr_debug("newsk=%p, subflow is mptcp=%d\n", newsk, sk_is_mptcp(newsk));  in mptcp_stream_accept()
3949 		struct mptcp_subflow_context *subflow;  in mptcp_stream_accept()  local
3952 		subflow = mptcp_subflow_ctx(newsk);  in mptcp_stream_accept()
3953 		new_mptcp_sock = subflow->conn;  in mptcp_stream_accept()
3955 		/* is_mptcp should be false if subflow->conn is missing, see  in mptcp_stream_accept()
3977 		mptcp_for_each_subflow(msk, subflow) {  in mptcp_stream_accept()
3978 			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_stream_accept()
3984 		/* Do late cleanup for the first subflow as necessary. Also  in mptcp_stream_accept()
4109 	struct mptcp_subflow_context *subflow;  in mptcp_napi_poll()  local
4113 	while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) {  in mptcp_napi_poll()
4114 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_napi_poll()
4118 			mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0));  in mptcp_napi_poll()
4122 			 * In both case must dequeue the subflow here - on the same  in mptcp_napi_poll()
4126 			clear_bit(MPTCP_DELEGATE_SCHEDULED, &subflow->delegated_status);  in mptcp_napi_poll()