70 	struct mptcp_subflow_context *subflow;  in __mptcp_socket_create()  local
81 	subflow = mptcp_subflow_ctx(ssock->sk);  in __mptcp_socket_create()
82 	list_add(&subflow->node, &msk->conn_list);  in __mptcp_socket_create()
84 	subflow->request_mptcp = 1;  in __mptcp_socket_create()
85 	subflow->subflow_id = msk->subflow_id++;  in __mptcp_socket_create()
88 	WRITE_ONCE(subflow->local_id, 0);  in __mptcp_socket_create()
343 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_move_skb()  local
365 	MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow);  in __mptcp_move_skb()
485 static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow)  in mptcp_timeout_from_subflow()  argument
487 	const struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_timeout_from_subflow()
489 	return inet_csk(ssk)->icsk_pending && !subflow->stale_count ?  in mptcp_timeout_from_subflow()
495 	struct mptcp_subflow_context *subflow;  in mptcp_set_timeout()  local
498 	mptcp_for_each_subflow(mptcp_sk(sk), subflow)  in mptcp_set_timeout()
499 		tout = max(tout, mptcp_timeout_from_subflow(subflow));  in mptcp_set_timeout()
526 	struct mptcp_subflow_context *subflow;  in mptcp_send_ack()  local
528 	mptcp_for_each_subflow(msk, subflow)  in mptcp_send_ack()
529 		mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow));  in mptcp_send_ack()
558 	struct mptcp_subflow_context *subflow;  in mptcp_cleanup_rbuf()  local
566 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_cleanup_rbuf()
567 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_cleanup_rbuf()
640 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_move_skbs_from_subflow()  local
668 		map_remaining = subflow->map_data_len -  in __mptcp_move_skbs_from_subflow()
669 				mptcp_subflow_get_map_offset(subflow);  in __mptcp_move_skbs_from_subflow()
688 			subflow->map_data_len = skb->len;  in __mptcp_move_skbs_from_subflow()
814 	struct mptcp_subflow_context *subflow;  in __mptcp_error_report()  local
817 	mptcp_for_each_subflow(msk, subflow)  in __mptcp_error_report()
818 		if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow)))  in __mptcp_error_report()
851 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_data_ready()  local
859 	if (unlikely(subflow->disposable))  in mptcp_data_ready()
908 	struct mptcp_subflow_context *tmp, *subflow;  in __mptcp_flush_join_list()  local
911 	list_for_each_entry_safe(subflow, tmp, join_list, node) {  in __mptcp_flush_join_list()
912 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_flush_join_list()
915 		list_move_tail(&subflow->node, &msk->conn_list);  in __mptcp_flush_join_list()
955 	struct mptcp_subflow_context *subflow;  in mptcp_subflow_recv_lookup()  local
959 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_subflow_recv_lookup()
960 		if (READ_ONCE(subflow->data_avail))  in mptcp_subflow_recv_lookup()
961 			return mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_recv_lookup()
1088 	struct mptcp_subflow_context *subflow;  in mptcp_enter_memory_pressure()  local
1092 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_enter_memory_pressure()
1093 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_enter_memory_pressure()
1396 void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow)  in mptcp_subflow_set_active()  argument
1398 	if (!subflow->stale)  in mptcp_subflow_set_active()
1401 	subflow->stale = 0;  in mptcp_subflow_set_active()
1402 	MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER);  in mptcp_subflow_set_active()
1405 bool mptcp_subflow_active(struct mptcp_subflow_context *subflow)  in mptcp_subflow_active()  argument
1407 	if (unlikely(subflow->stale)) {  in mptcp_subflow_active()
1408 		u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp);  in mptcp_subflow_active()
1410 		if (subflow->stale_rcv_tstamp == rcv_tstamp)  in mptcp_subflow_active()
1413 		mptcp_subflow_set_active(subflow);  in mptcp_subflow_active()
1415 	return __mptcp_subflow_active(subflow);  in mptcp_subflow_active()
1429 	struct mptcp_subflow_context *subflow;  in mptcp_subflow_get_send()  local
1443 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_subflow_get_send()
1444 		bool backup = subflow->backup || subflow->request_bkup;  in mptcp_subflow_get_send()
1446 		trace_mptcp_subflow_get_send(subflow);  in mptcp_subflow_get_send()
1447 		ssk =  mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_send()
1448 		if (!mptcp_subflow_active(subflow))  in mptcp_subflow_get_send()
1451 		tout = max(tout, mptcp_timeout_from_subflow(subflow));  in mptcp_subflow_get_send()
1453 		pace = subflow->avg_pacing_rate;  in mptcp_subflow_get_send()
1456 			subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate);  in mptcp_subflow_get_send()
1457 			pace = subflow->avg_pacing_rate;  in mptcp_subflow_get_send()
1494 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_get_send()
1495 	subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem +  in mptcp_subflow_get_send()
1599 		struct mptcp_subflow_context *subflow;  in __mptcp_push_pending()  local
1607 		mptcp_for_each_subflow(msk, subflow) {  in __mptcp_push_pending()
1608 			if (READ_ONCE(subflow->scheduled)) {  in __mptcp_push_pending()
1609 				mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_push_pending()
1612 				ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_push_pending()
1665 		struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_subflow_push_pending()  local
1672 			mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_subflow_push_pending()
1684 		if (READ_ONCE(subflow->scheduled)) {  in __mptcp_subflow_push_pending()
1685 			mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_subflow_push_pending()
1692 		mptcp_for_each_subflow(msk, subflow) {  in __mptcp_subflow_push_pending()
1693 			if (READ_ONCE(subflow->scheduled)) {  in __mptcp_subflow_push_pending()
1694 				xmit_ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_subflow_push_pending()
1696 					mptcp_subflow_delegate(subflow,  in __mptcp_subflow_push_pending()
2010 	struct mptcp_subflow_context *subflow;  in mptcp_rcv_space_adjust()  local
2034 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_rcv_space_adjust()
2039 		tp = tcp_sk(mptcp_subflow_tcp_sock(subflow));  in mptcp_rcv_space_adjust()
2083 			mptcp_for_each_subflow(msk, subflow) {  in mptcp_rcv_space_adjust()
2087 				ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_rcv_space_adjust()
2340 	struct mptcp_subflow_context *subflow;  in mptcp_subflow_get_retrans()  local
2343 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_subflow_get_retrans()
2344 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_retrans()
2346 		if (!__mptcp_subflow_active(subflow))  in mptcp_subflow_get_retrans()
2352 			min_stale_count = min_t(int, min_stale_count, subflow->stale_count);  in mptcp_subflow_get_retrans()
2356 		if (subflow->backup || subflow->request_bkup) {  in mptcp_subflow_get_retrans()
2419 				       struct mptcp_subflow_context *subflow,  in __mptcp_subflow_disconnect()  argument
2428 		mptcp_subflow_ctx_reset(subflow);  in __mptcp_subflow_disconnect()
2443 			      struct mptcp_subflow_context *subflow,  in __mptcp_close_ssk()  argument
2466 		list_del(&subflow->node);  in __mptcp_close_ssk()
2476 		subflow->send_fastclose = 1;  in __mptcp_close_ssk()
2481 		__mptcp_subflow_disconnect(ssk, subflow, flags);  in __mptcp_close_ssk()
2487 	subflow->disposable = 1;  in __mptcp_close_ssk()
2495 		kfree_rcu(subflow, rcu);  in __mptcp_close_ssk()
2536 		     struct mptcp_subflow_context *subflow)  in mptcp_close_ssk()  argument
2539 	if (subflow->close_event_done)  in mptcp_close_ssk()
2542 	subflow->close_event_done = true;  in mptcp_close_ssk()
2550 	mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow);  in mptcp_close_ssk()
2552 	__mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH);  in mptcp_close_ssk()
2562 	struct mptcp_subflow_context *subflow, *tmp;  in __mptcp_close_subflow()  local
2567 	mptcp_for_each_subflow_safe(msk, subflow, tmp) {  in __mptcp_close_subflow()
2568 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_close_subflow()
2580 		mptcp_close_ssk(sk, ssk, subflow);  in __mptcp_close_subflow()
2597 	struct mptcp_subflow_context *subflow, *tmp;  in mptcp_check_fastclose()  local
2605 	mptcp_for_each_subflow_safe(msk, subflow, tmp) {  in mptcp_check_fastclose()
2606 		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);  in mptcp_check_fastclose()
2647 	struct mptcp_subflow_context *subflow;  in __mptcp_retrans()  local
2679 	mptcp_for_each_subflow(msk, subflow) {  in __mptcp_retrans()
2680 		if (READ_ONCE(subflow->scheduled)) {  in __mptcp_retrans()
2683 			mptcp_subflow_set_scheduled(subflow, false);  in __mptcp_retrans()
2685 			ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_retrans()
2763 	struct mptcp_subflow_context *subflow, *tmp;  in mptcp_do_fastclose()  local
2767 	mptcp_for_each_subflow_safe(msk, subflow, tmp)  in mptcp_do_fastclose()
2768 		__mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow),  in mptcp_do_fastclose()
2769 				  subflow, MPTCP_CF_FASTCLOSE);  in mptcp_do_fastclose()
3002 	struct mptcp_subflow_context *subflow;  in mptcp_check_send_data_fin()  local
3018 	mptcp_for_each_subflow(msk, subflow) {  in mptcp_check_send_data_fin()
3019 		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);  in mptcp_check_send_data_fin()
3097 	struct mptcp_subflow_context *subflow;  in __mptcp_close()  local
3124 	mptcp_for_each_subflow(msk, subflow) {  in __mptcp_close()
3125 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_close()
3134 			subflow->fail_tout = 0;  in __mptcp_close()
3307 	struct mptcp_subflow_context *subflow;  in mptcp_sk_clone_init()  local
3355 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_sk_clone_init()
3356 	list_add(&subflow->node, &msk->conn_list);  in mptcp_sk_clone_init()
3373 		__mptcp_subflow_fully_established(msk, subflow, mp_opt);  in mptcp_sk_clone_init()
3399 	struct mptcp_subflow_context *subflow, *tmp;  in mptcp_destroy_common()  local
3405 	mptcp_for_each_subflow_safe(msk, subflow, tmp)  in mptcp_destroy_common()
3406 		__mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags);  in mptcp_destroy_common()
3542 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_subflow_process_delegated()  local
3543 	struct sock *sk = subflow->conn;  in mptcp_subflow_process_delegated()
3592 	struct mptcp_subflow_context *subflow;  in mptcp_finish_connect()  local
3596 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_finish_connect()
3597 	sk = subflow->conn;  in mptcp_finish_connect()
3600 	pr_debug("msk=%p, token=%u\n", sk, subflow->token);  in mptcp_finish_connect()
3602 	subflow->map_seq = subflow->iasn;  in mptcp_finish_connect()
3603 	subflow->map_subflow_seq = 1;  in mptcp_finish_connect()
3608 	WRITE_ONCE(msk->local_key, subflow->local_key);  in mptcp_finish_connect()
3624 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_finish_join()  local
3625 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);  in mptcp_finish_join()
3629 	pr_debug("msk=%p, subflow=%p\n", msk, subflow);  in mptcp_finish_join()
3633 		subflow->reset_reason = MPTCP_RST_EMPTCP;  in mptcp_finish_join()
3638 	if (!list_empty(&subflow->node)) {  in mptcp_finish_join()
3655 			list_add_tail(&subflow->node, &msk->conn_list);  in mptcp_finish_join()
3659 		list_add_tail(&subflow->node, &msk->join_list);  in mptcp_finish_join()
3666 		subflow->reset_reason = MPTCP_RST_EPROHIBIT;  in mptcp_finish_join()
3750 	struct mptcp_subflow_context *subflow;  in mptcp_connect()  local
3760 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_connect()
3766 		mptcp_subflow_early_fallback(msk, subflow);  in mptcp_connect()
3768 	if (subflow->request_mptcp) {  in mptcp_connect()
3771 			mptcp_subflow_early_fallback(msk, subflow);  in mptcp_connect()
3774 			mptcp_subflow_early_fallback(msk, subflow);  in mptcp_connect()
3778 	WRITE_ONCE(msk->write_seq, subflow->idsn);  in mptcp_connect()
3779 	WRITE_ONCE(msk->snd_nxt, subflow->idsn);  in mptcp_connect()
3780 	WRITE_ONCE(msk->snd_una, subflow->idsn);  in mptcp_connect()
3949 		struct mptcp_subflow_context *subflow;  in mptcp_stream_accept()  local
3952 		subflow = mptcp_subflow_ctx(newsk);  in mptcp_stream_accept()
3953 		new_mptcp_sock = subflow->conn;  in mptcp_stream_accept()
3977 		mptcp_for_each_subflow(msk, subflow) {  in mptcp_stream_accept()
3978 			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_stream_accept()
4109 	struct mptcp_subflow_context *subflow;  in mptcp_napi_poll()  local
4113 	while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) {  in mptcp_napi_poll()
4114 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_napi_poll()
4118 			mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0));  in mptcp_napi_poll()
4126 			clear_bit(MPTCP_DELEGATE_SCHEDULED, &subflow->delegated_status);  in mptcp_napi_poll()