23 static void rxrpc_rack_mark_lost(struct rxrpc_call *call,  in rxrpc_rack_mark_lost()  argument
28 			call->tx_nr_resent--;  in rxrpc_rack_mark_lost()
30 		call->tx_nr_lost++;  in rxrpc_rack_mark_lost()
62 static void rxrpc_rack_update(struct rxrpc_call *call,  in rxrpc_rack_update()  argument
69 	ktime_t rtt = ktime_sub(call->acks_latest_ts, xmit_ts);  in rxrpc_rack_update()
72 		call->tx_nr_lost--;  in rxrpc_rack_update()
76 		if (before(call->acks_highest_serial, tq->segment_serial[ix]))  in rxrpc_rack_update()
78 		if (rtt < minmax_get(&call->min_rtt))  in rxrpc_rack_update()
88 	if (ktime_after(xmit_ts, call->rack_rtt_ts)) {  in rxrpc_rack_update()
89 		call->rack_rtt	  = rtt;  in rxrpc_rack_update()
90 		call->rack_rtt_ts = xmit_ts;  in rxrpc_rack_update()
93 	if (rxrpc_rack_sent_after(xmit_ts, seq, call->rack_xmit_ts, call->rack_end_seq)) {  in rxrpc_rack_update()
94 		call->rack_rtt = rtt;  in rxrpc_rack_update()
95 		call->rack_xmit_ts = xmit_ts;  in rxrpc_rack_update()
96 		call->rack_end_seq = seq;  in rxrpc_rack_update()
103 static void rxrpc_rack_detect_reordering(struct rxrpc_call *call,  in rxrpc_rack_detect_reordering()  argument
114 	if (after(seq, call->rack_fack))  in rxrpc_rack_detect_reordering()
115 		call->rack_fack = seq;  in rxrpc_rack_detect_reordering()
116 	else if (before(seq, call->rack_fack) &&  in rxrpc_rack_detect_reordering()
118 		call->rack_reordering_seen = true;  in rxrpc_rack_detect_reordering()
121 void rxrpc_input_rack_one(struct rxrpc_call *call,  in rxrpc_input_rack_one()  argument
126 	rxrpc_rack_update(call, summary, tq, ix);  in rxrpc_input_rack_one()
127 	rxrpc_rack_detect_reordering(call, summary, tq, ix);  in rxrpc_input_rack_one()
130 void rxrpc_input_rack(struct rxrpc_call *call,  in rxrpc_input_rack()  argument
139 		rxrpc_input_rack_one(call, summary, tq, ix);  in rxrpc_input_rack()
142 	trace_rxrpc_rack_update(call, summary);  in rxrpc_input_rack()
156 static ktime_t rxrpc_rack_update_reo_wnd(struct rxrpc_call *call,  in rxrpc_rack_update_reo_wnd()  argument
159 	rxrpc_seq_t snd_una = call->acks_lowest_nak; /* Lowest unack'd seq */  in rxrpc_rack_update_reo_wnd()
160 	rxrpc_seq_t snd_nxt = call->tx_transmitted + 1; /* Next seq to be sent */  in rxrpc_rack_update_reo_wnd()
165 	if (!call->rack_dsack_round_none &&  in rxrpc_rack_update_reo_wnd()
166 	    after_eq(snd_una, call->rack_dsack_round))  in rxrpc_rack_update_reo_wnd()
167 		call->rack_dsack_round_none = true;  in rxrpc_rack_update_reo_wnd()
172 	if (call->rack_dsack_round_none && have_dsack_option) {  in rxrpc_rack_update_reo_wnd()
173 		call->rack_dsack_round_none = false;  in rxrpc_rack_update_reo_wnd()
174 		call->rack_dsack_round = snd_nxt;  in rxrpc_rack_update_reo_wnd()
175 		call->rack_reo_wnd_mult++;  in rxrpc_rack_update_reo_wnd()
176 		call->rack_reo_wnd_persist = 16;  in rxrpc_rack_update_reo_wnd()
178 		call->rack_reo_wnd_persist--;  in rxrpc_rack_update_reo_wnd()
179 		if (call->rack_reo_wnd_persist <= 0)  in rxrpc_rack_update_reo_wnd()
180 			call->rack_reo_wnd_mult = 1;  in rxrpc_rack_update_reo_wnd()
183 	if (!call->rack_reordering_seen) {  in rxrpc_rack_update_reo_wnd()
186 		if (call->acks_nr_sacks >= dup_thresh)  in rxrpc_rack_update_reo_wnd()
190 	return us_to_ktime(umin(call->rack_reo_wnd_mult * minmax_get(&call->min_rtt) / 4,  in rxrpc_rack_update_reo_wnd()
191 				call->srtt_us >> 3));  in rxrpc_rack_update_reo_wnd()
197 static ktime_t rxrpc_rack_detect_loss(struct rxrpc_call *call,  in rxrpc_rack_detect_loss()  argument
203 	call->rack_reo_wnd = rxrpc_rack_update_reo_wnd(call, summary);  in rxrpc_rack_detect_loss()
204 	lost_after = ktime_add(call->rack_rtt, call->rack_reo_wnd);  in rxrpc_rack_detect_loss()
205 	trace_rxrpc_rack_scan_loss(call);  in rxrpc_rack_detect_loss()
207 	for (tq = call->tx_queue; tq; tq = tq->next) {  in rxrpc_rack_detect_loss()
210 		if (after(tq->qbase, call->tx_transmitted))  in rxrpc_rack_detect_loss()
212 		trace_rxrpc_rack_scan_loss_tq(call, tq, nacks);  in rxrpc_rack_detect_loss()
225 			if (rxrpc_rack_sent_after(call->rack_xmit_ts, call->rack_end_seq,  in rxrpc_rack_detect_loss()
229 					rxrpc_rack_mark_lost(call, tq, ix);  in rxrpc_rack_detect_loss()
230 					trace_rxrpc_rack_detect_loss(call, summary, seq);  in rxrpc_rack_detect_loss()
244 void rxrpc_rack_detect_loss_and_arm_timer(struct rxrpc_call *call,  in rxrpc_rack_detect_loss_and_arm_timer()  argument
247 	ktime_t timeout = rxrpc_rack_detect_loss(call, summary);  in rxrpc_rack_detect_loss_and_arm_timer()
250 		call->rack_timer_mode = RXRPC_CALL_RACKTIMER_RACK_REORDER;  in rxrpc_rack_detect_loss_and_arm_timer()
251 		call->rack_timo_at = ktime_add(ktime_get_real(), timeout);  in rxrpc_rack_detect_loss_and_arm_timer()
252 		trace_rxrpc_rack_timer(call, timeout, false);  in rxrpc_rack_detect_loss_and_arm_timer()
253 		trace_rxrpc_timer_set(call, timeout, rxrpc_timer_trace_rack_reo);  in rxrpc_rack_detect_loss_and_arm_timer()
260 static void rxrpc_rack_mark_losses_on_rto(struct rxrpc_call *call)  in rxrpc_rack_mark_losses_on_rto()  argument
263 	rxrpc_seq_t snd_una = call->acks_lowest_nak; /* Lowest unack'd seq */  in rxrpc_rack_mark_losses_on_rto()
264 	ktime_t lost_after = ktime_add(call->rack_rtt, call->rack_reo_wnd);  in rxrpc_rack_mark_losses_on_rto()
267 	for (tq = call->tx_queue; tq; tq = tq->next) {  in rxrpc_rack_mark_losses_on_rto()
270 		trace_rxrpc_rack_mark_loss_tq(call, tq);  in rxrpc_rack_mark_losses_on_rto()
276 			if (after(seq, call->tx_transmitted))  in rxrpc_rack_mark_losses_on_rto()
282 				rxrpc_rack_mark_lost(call, tq, ix);  in rxrpc_rack_mark_losses_on_rto()
290 ktime_t rxrpc_tlp_calc_pto(struct rxrpc_call *call, ktime_t now)  in rxrpc_tlp_calc_pto()  argument
292 	unsigned int flight_size = rxrpc_tx_in_flight(call);  in rxrpc_tlp_calc_pto()
293 	ktime_t rto_at = ktime_add(call->tx_last_sent,  in rxrpc_tlp_calc_pto()
294 				   rxrpc_get_rto_backoff(call, false));  in rxrpc_tlp_calc_pto()
297 	if (call->rtt_count > 0) {  in rxrpc_tlp_calc_pto()
299 		pto = ns_to_ktime(call->srtt_us * NSEC_PER_USEC / 4);  in rxrpc_tlp_calc_pto()
301 			pto = ktime_add(pto, call->tlp_max_ack_delay);  in rxrpc_tlp_calc_pto()
314 void rxrpc_tlp_send_probe(struct rxrpc_call *call)  in rxrpc_tlp_send_probe()  argument
316 	unsigned int in_flight = rxrpc_tx_in_flight(call);  in rxrpc_tlp_send_probe()
318 	if (after_eq(call->acks_hard_ack, call->tx_transmitted))  in rxrpc_tlp_send_probe()
325 	if (!call->tlp_serial &&  in rxrpc_tlp_send_probe()
326 	    call->tlp_rtt_taken != call->rtt_taken) {  in rxrpc_tlp_send_probe()
327 		call->tlp_is_retrans = false;  in rxrpc_tlp_send_probe()
328 		if (after(call->send_top, call->tx_transmitted) &&  in rxrpc_tlp_send_probe()
329 		    rxrpc_tx_window_space(call) > 0) {  in rxrpc_tlp_send_probe()
331 			call->tx_last_serial = 0;  in rxrpc_tlp_send_probe()
332 			rxrpc_transmit_some_data(call, 1, rxrpc_txdata_tlp_new_data);  in rxrpc_tlp_send_probe()
333 			call->tlp_serial = call->tx_last_serial;  in rxrpc_tlp_send_probe()
334 			call->tlp_seq = call->tx_transmitted;  in rxrpc_tlp_send_probe()
335 			trace_rxrpc_tlp_probe(call, rxrpc_tlp_probe_trace_transmit_new);  in rxrpc_tlp_send_probe()
336 			in_flight = rxrpc_tx_in_flight(call);  in rxrpc_tlp_send_probe()
339 			call->tx_last_serial = 0;  in rxrpc_tlp_send_probe()
340 			rxrpc_resend_tlp(call);  in rxrpc_tlp_send_probe()
341 			call->tlp_is_retrans = true;  in rxrpc_tlp_send_probe()
342 			trace_rxrpc_tlp_probe(call, rxrpc_tlp_probe_trace_retransmit);  in rxrpc_tlp_send_probe()
345 		trace_rxrpc_tlp_probe(call, rxrpc_tlp_probe_trace_busy);  in rxrpc_tlp_send_probe()
349 		ktime_t rto = rxrpc_get_rto_backoff(call, false);  in rxrpc_tlp_send_probe()
351 		call->rack_timer_mode = RXRPC_CALL_RACKTIMER_RTO;  in rxrpc_tlp_send_probe()
352 		call->rack_timo_at = ktime_add(ktime_get_real(), rto);  in rxrpc_tlp_send_probe()
353 		trace_rxrpc_rack_timer(call, rto, false);  in rxrpc_tlp_send_probe()
354 		trace_rxrpc_timer_set(call, rto, rxrpc_timer_trace_rack_rto);  in rxrpc_tlp_send_probe()
361 void rxrpc_tlp_process_ack(struct rxrpc_call *call, struct rxrpc_ack_summary *summary)  in rxrpc_tlp_process_ack()  argument
363 	if (!call->tlp_serial || after(call->tlp_seq, call->acks_hard_ack))  in rxrpc_tlp_process_ack()
366 	if (!call->tlp_is_retrans) {  in rxrpc_tlp_process_ack()
368 		trace_rxrpc_tlp_ack(call, summary, rxrpc_tlp_ack_trace_new_data);  in rxrpc_tlp_process_ack()
369 		call->tlp_serial = 0;  in rxrpc_tlp_process_ack()
371 		   summary->acked_serial == call->tlp_serial) {  in rxrpc_tlp_process_ack()
373 		trace_rxrpc_tlp_ack(call, summary, rxrpc_tlp_ack_trace_dup_acked);  in rxrpc_tlp_process_ack()
374 		call->tlp_serial = 0;  in rxrpc_tlp_process_ack()
375 	} else if (after(call->acks_hard_ack, call->tlp_seq)) {  in rxrpc_tlp_process_ack()
377 		trace_rxrpc_tlp_ack(call, summary, rxrpc_tlp_ack_trace_hard_beyond);  in rxrpc_tlp_process_ack()
378 		call->tlp_serial = 0;  in rxrpc_tlp_process_ack()
382 		trace_rxrpc_tlp_ack(call, summary, rxrpc_tlp_ack_trace_acked);  in rxrpc_tlp_process_ack()
386 		call->tlp_serial = 0;  in rxrpc_tlp_process_ack()
388 		trace_rxrpc_tlp_ack(call, summary, rxrpc_tlp_ack_trace_incomplete);  in rxrpc_tlp_process_ack()
395 void rxrpc_rack_timer_expired(struct rxrpc_call *call, ktime_t overran_by)  in rxrpc_rack_timer_expired()  argument
398 	enum rxrpc_rack_timer_mode mode = call->rack_timer_mode;  in rxrpc_rack_timer_expired()
400 	trace_rxrpc_rack_timer(call, overran_by, true);  in rxrpc_rack_timer_expired()
401 	call->rack_timer_mode = RXRPC_CALL_RACKTIMER_OFF;  in rxrpc_rack_timer_expired()
405 		rxrpc_rack_detect_loss_and_arm_timer(call, &summary);  in rxrpc_rack_timer_expired()
408 		rxrpc_tlp_send_probe(call);  in rxrpc_rack_timer_expired()
412 		rxrpc_rack_mark_losses_on_rto(call);  in rxrpc_rack_timer_expired()
416 		pr_warn("Unexpected rack timer %u", call->rack_timer_mode);  in rxrpc_rack_timer_expired()