1#!/usr/bin/env python 2# @lint-avoid-python-3-compatibility-imports 3# 4# tcpcong Measure tcp congestion control status duration. 5# For Linux, uses BCC, eBPF. 6# 7# USAGE: tcpcong [-h] [-T] [-L] [-R] [-m] [-d] [interval] [outputs] 8# 9# Copyright (c) Ping Gan. 10# 11# 27-Jan-2022 Ping Gan Created this. 12 13from __future__ import print_function 14from bcc import BPF 15from time import sleep, strftime 16from struct import pack 17from socket import inet_ntop, AF_INET, AF_INET6 18from struct import pack 19import argparse 20 21examples = """examples: 22 ./tcpcong # show tcp congestion status duration 23 ./tcpcong 1 10 # show 1 second summaries, 10 times 24 ./tcpcong -L 3000-3006 1 # 1s summaries, local port 3000-3006 25 ./tcpcong -R 5000-5005 1 # 1s summaries, remote port 5000-5005 26 ./tcpcong -uT 1 # 1s summaries, microseconds, and timestamps 27 ./tcpcong -d # show the duration as histograms 28""" 29 30parser = argparse.ArgumentParser( 31 description="Summarize tcp socket congestion control status duration", 32 formatter_class=argparse.RawDescriptionHelpFormatter, 33 epilog=examples) 34parser.add_argument("-L", "--localport", 35 help="trace local ports only") 36parser.add_argument("-R", "--remoteport", 37 help="trace the dest ports only") 38parser.add_argument("-T", "--timestamp", action="store_true", 39 help="include timestamp on output") 40parser.add_argument("-d", "--dist", action="store_true", 41 help="show distributions as histograms") 42parser.add_argument("-u", "--microseconds", action="store_true", 43 help="output in microseconds") 44parser.add_argument("interval", nargs="?", default=99999999, 45 help="output interval, in seconds") 46parser.add_argument("outputs", nargs="?", default=99999999, 47 help="number of outputs") 48parser.add_argument("--ebpf", action="store_true", 49 help=argparse.SUPPRESS) 50args = parser.parse_args() 51countdown = int(args.outputs) 52debug = 0 53 54start_rport = end_rport = -1 55if args.remoteport: 56 rports = args.remoteport.split("-") 57 if (len(rports) != 2) and (len(rports) != 1): 58 print("unrecognized remote port range") 59 exit(1) 60 if len(rports) == 2: 61 start_rport = int(rports[0]) 62 end_rport = int(rports[1]) 63 else: 64 start_rport = int(rports[0]) 65 end_rport = int(rports[0]) 66if start_rport > end_rport: 67 tmp = start_rport 68 start_rport = end_rport 69 end_rport = tmp 70 71start_lport = end_lport = -1 72if args.localport: 73 lports = args.localport.split("-") 74 if (len(lports) != 2) and (len(lports) != 1): 75 print("unrecognized local port range") 76 exit(1) 77 if len(lports) == 2: 78 start_lport = int(lports[0]) 79 end_lport = int(lports[1]) 80 else: 81 start_lport = int(lports[0]) 82 end_lport = int(lports[0]) 83if start_lport > end_lport: 84 tmp = start_lport 85 start_lport = end_lport 86 end_lport = tmp 87 88# define BPF program 89bpf_head_text = """ 90#include <uapi/linux/ptrace.h> 91#include <net/sock.h> 92#include <bcc/proto.h> 93#include <net/tcp.h> 94#include <net/inet_connection_sock.h> 95 96typedef struct ipv4_flow_key { 97 u32 saddr; 98 u32 daddr; 99 u16 lport; 100 u16 dport; 101} ipv4_flow_key_t; 102 103typedef struct ipv6_flow_key { 104 unsigned __int128 saddr; 105 unsigned __int128 daddr; 106 u16 lport; 107 u16 dport; 108} ipv6_flow_key_t; 109 110typedef struct data_val { 111 DEF_TEXT 112 u64 last_ts; 113 u16 last_cong_stat; 114} data_val_t; 115 116BPF_HASH(ipv4_stat, ipv4_flow_key_t, data_val_t); 117BPF_HASH(ipv6_stat, ipv6_flow_key_t, data_val_t); 118 119HIST_TABLE 120""" 121 122bpf_extra_head = """ 123typedef struct process_key { 124 char comm[TASK_COMM_LEN]; 125 u32 tid; 126} process_key_t; 127 128typedef struct ipv4_flow_val { 129 ipv4_flow_key_t ipv4_key; 130 u16 cong_state; 131} ipv4_flow_val_t; 132 133typedef struct ipv6_flow_val { 134 ipv6_flow_key_t ipv6_key; 135 u16 cong_state; 136} ipv6_flow_val_t; 137 138BPF_HASH(start_ipv4, process_key_t, ipv4_flow_val_t); 139BPF_HASH(start_ipv6, process_key_t, ipv6_flow_val_t); 140SOCK_STORE_DEF 141 142typedef struct cong { 143 u8 cong_stat:5, 144 ca_inited:1, 145 ca_setsockopt:1, 146 ca_dstlocked:1; 147} cong_status_t; 148""" 149 150bpf_no_ca_tp_body_text = """ 151static int entry_state_update_func(struct sock *sk) 152{ 153 u16 dport = 0, lport = 0; 154 u32 tid = bpf_get_current_pid_tgid(); 155 process_key_t key = {0}; 156 bpf_get_current_comm(&key.comm, sizeof(key.comm)); 157 key.tid = tid; 158 159 u64 family = sk->__sk_common.skc_family; 160 struct inet_connection_sock *icsk = inet_csk(sk); 161 cong_status_t cong_status; 162 bpf_probe_read_kernel(&cong_status, sizeof(cong_status), 163 (void *)((long)&icsk->icsk_retransmits) - 1); 164 if (family == AF_INET) { 165 ipv4_flow_val_t ipv4_val = {0}; 166 ipv4_val.ipv4_key.saddr = sk->__sk_common.skc_rcv_saddr; 167 ipv4_val.ipv4_key.daddr = sk->__sk_common.skc_daddr; 168 ipv4_val.ipv4_key.lport = sk->__sk_common.skc_num; 169 dport = sk->__sk_common.skc_dport; 170 dport = ntohs(dport); 171 lport = ipv4_val.ipv4_key.lport; 172 FILTER_LPORT 173 FILTER_DPORT 174 ipv4_val.ipv4_key.dport = dport; 175 ipv4_val.cong_state = cong_status.cong_stat + 1; 176 start_ipv4.update(&key, &ipv4_val); 177 } else if (family == AF_INET6) { 178 ipv6_flow_val_t ipv6_val = {0}; 179 bpf_probe_read_kernel(&ipv6_val.ipv6_key.saddr, 180 sizeof(ipv6_val.ipv6_key.saddr), 181 &sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); 182 bpf_probe_read_kernel(&ipv6_val.ipv6_key.daddr, 183 sizeof(ipv6_val.ipv6_key.daddr), 184 &sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32); 185 ipv6_val.ipv6_key.lport = sk->__sk_common.skc_num; 186 dport = sk->__sk_common.skc_dport; 187 dport = ntohs(dport); 188 lport = ipv6_val.ipv6_key.lport; 189 FILTER_LPORT 190 FILTER_DPORT 191 ipv6_val.ipv6_key.dport = dport; 192 ipv6_val.cong_state = cong_status.cong_stat + 1; 193 start_ipv6.update(&key, &ipv6_val); 194 } 195 SOCK_STORE_ADD 196 return 0; 197} 198 199static int ret_state_update_func(struct sock *sk) 200{ 201 u64 ts, ts1; 202 u16 family, last_cong_state; 203 u16 dport = 0, lport = 0; 204 u32 tid = bpf_get_current_pid_tgid(); 205 process_key_t key = {0}; 206 bpf_get_current_comm(&key.comm, sizeof(key.comm)); 207 key.tid = tid; 208 209 struct inet_connection_sock *icsk = inet_csk(sk); 210 cong_status_t cong_status; 211 bpf_probe_read_kernel(&cong_status, sizeof(cong_status), 212 (void *)((long)&icsk->icsk_retransmits) - 1); 213 data_val_t *datap, data = {0}; 214 STATE_KEY 215 bpf_probe_read_kernel(&family, sizeof(family), 216 &sk->__sk_common.skc_family); 217 if (family == AF_INET) { 218 ipv4_flow_val_t *val4 = start_ipv4.lookup(&key); 219 if (val4 == 0) { 220 SOCK_STORE_DEL 221 return 0; //missed 222 } 223 ipv4_flow_key_t keyv4 = {0}; 224 bpf_probe_read_kernel(&keyv4, sizeof(ipv4_flow_key_t), 225 &(val4->ipv4_key)); 226 dport = keyv4.dport; 227 lport = keyv4.lport; 228 FILTER_LPORT 229 FILTER_DPORT 230 datap = ipv4_stat.lookup(&keyv4); 231 if (datap == 0) { 232 data.last_ts = bpf_ktime_get_ns(); 233 data.last_cong_stat = val4->cong_state; 234 ipv4_stat.update(&keyv4, &data); 235 } else { 236 last_cong_state = val4->cong_state; 237 if ((cong_status.cong_stat + 1) != last_cong_state) { 238 ts1 = bpf_ktime_get_ns(); 239 ts = ts1 - datap->last_ts; 240 datap->last_ts = ts1; 241 datap->last_cong_stat = cong_status.cong_stat + 1; 242 ts /= 1000; 243 STORE 244 } 245 } 246 start_ipv4.delete(&key); 247 } else if (family == AF_INET6) { 248 ipv6_flow_val_t *val6 = start_ipv6.lookup(&key); 249 if (val6 == 0) { 250 SOCK_STORE_DEL 251 return 0; //missed 252 } 253 ipv6_flow_key_t keyv6 = {0}; 254 bpf_probe_read_kernel(&keyv6, sizeof(ipv6_flow_key_t), 255 &(val6->ipv6_key)); 256 dport = keyv6.dport; 257 lport = keyv6.lport; 258 FILTER_LPORT 259 FILTER_DPORT 260 datap = ipv6_stat.lookup(&keyv6); 261 if (datap == 0) { 262 data.last_ts = bpf_ktime_get_ns(); 263 data.last_cong_stat = val6->cong_state; 264 ipv6_stat.update(&keyv6, &data); 265 } else { 266 last_cong_state = val6->cong_state; 267 if ((cong_status.cong_stat + 1) != last_cong_state) { 268 ts1 = bpf_ktime_get_ns(); 269 ts = ts1 - datap->last_ts; 270 datap->last_ts = ts1; 271 datap->last_cong_stat = (cong_status.cong_stat + 1); 272 ts /= 1000; 273 STORE 274 } 275 } 276 start_ipv6.delete(&key); 277 } 278 SOCK_STORE_DEL 279 return 0; 280} 281""" 282 283bpf_ca_tp_body_text = """ 284TRACEPOINT_PROBE(tcp, tcp_cong_state_set) 285{ 286 u64 ts, ts1; 287 u16 family, last_cong_state, dport = 0, lport = 0; 288 u8 cong_state; 289 const struct sock *sk = (const struct sock *)args->skaddr; 290 data_val_t *datap, data = {0}; 291 292 family = sk->__sk_common.skc_family; 293 dport = args->dport; 294 lport = args->sport; 295 cong_state = args->cong_state; 296 STATE_KEY 297 if (family == AF_INET) { 298 ipv4_flow_key_t key4 = {0}; 299 key4.saddr = sk->__sk_common.skc_rcv_saddr; 300 key4.daddr = sk->__sk_common.skc_daddr; 301 FILTER_LPORT 302 FILTER_DPORT 303 key4.lport = lport; 304 key4.dport = dport; 305 datap = ipv4_stat.lookup(&key4); 306 if (datap == 0) { 307 data.last_ts = bpf_ktime_get_ns(); 308 data.last_cong_stat = cong_state + 1; 309 ipv4_stat.update(&key4, &data); 310 } else { 311 last_cong_state = datap->last_cong_stat; 312 if ((cong_state + 1) != last_cong_state) { 313 ts1 = bpf_ktime_get_ns(); 314 ts = ts1 - datap->last_ts; 315 datap->last_ts = ts1; 316 datap->last_cong_stat = cong_state + 1; 317 ts /= 1000; 318 STORE 319 } 320 } 321 } else if (family == AF_INET6) { 322 ipv6_flow_key_t key6 = {0}; 323 bpf_probe_read_kernel(&key6.saddr, sizeof(key6.saddr), 324 &sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); 325 bpf_probe_read_kernel(&key6.daddr, sizeof(key6.daddr), 326 &sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32); 327 FILTER_LPORT 328 FILTER_DPORT 329 key6.lport = lport; 330 key6.dport = dport; 331 datap = ipv6_stat.lookup(&key6); 332 if (datap == 0) { 333 data.last_ts = bpf_ktime_get_ns(); 334 data.last_cong_stat = cong_state + 1; 335 ipv6_stat.update(&key6, &data); 336 } else { 337 last_cong_state = datap->last_cong_stat; 338 if ((cong_state + 1) != last_cong_state) { 339 ts1 = bpf_ktime_get_ns(); 340 ts = ts1 - datap->last_ts; 341 datap->last_ts = ts1; 342 datap->last_cong_stat = cong_state + 1; 343 ts /= 1000; 344 STORE 345 } 346 } 347 } 348 return 0; 349} 350""" 351 352kprobe_program = """ 353int entry_func(struct pt_regs *ctx, struct sock *sk) 354{ 355 return entry_state_update_func(sk); 356} 357 358int ret_func(struct pt_regs *ctx) 359{ 360 u32 tid = bpf_get_current_pid_tgid(); 361 process_key_t key = {0}; 362 bpf_get_current_comm(&key.comm, sizeof(key.comm)); 363 key.tid = tid; 364 struct sock **sockpp; 365 sockpp = sock_store.lookup(&key); 366 if (sockpp == 0) { 367 return 0; //miss the entry 368 } 369 struct sock *sk = *sockpp; 370 return ret_state_update_func(sk); 371} 372""" 373 374kfunc_program = """ 375KFUNC_PROBE(tcp_fastretrans_alert, struct sock *sk) 376{ 377 return entry_state_update_func(sk); 378} 379 380KRETFUNC_PROBE(tcp_fastretrans_alert, struct sock *sk) 381{ 382 return ret_state_update_func(sk); 383} 384 385KFUNC_PROBE(tcp_enter_cwr, struct sock *sk) 386{ 387 return entry_state_update_func(sk); 388} 389 390KRETFUNC_PROBE(tcp_enter_cwr, struct sock *sk) 391{ 392 return ret_state_update_func(sk); 393} 394 395KFUNC_PROBE(tcp_enter_loss, struct sock *sk) 396{ 397 return entry_state_update_func(sk); 398} 399 400KRETFUNC_PROBE(tcp_enter_loss, struct sock *sk) 401{ 402 return ret_state_update_func(sk); 403} 404 405KFUNC_PROBE(tcp_enter_recovery, struct sock *sk) 406{ 407 return entry_state_update_func(sk); 408} 409 410KRETFUNC_PROBE(tcp_enter_recovery, struct sock *sk) 411{ 412 return ret_state_update_func(sk); 413} 414 415KFUNC_PROBE(tcp_process_tlp_ack, struct sock *sk) 416{ 417 return entry_state_update_func(sk); 418} 419 420KRETFUNC_PROBE(tcp_process_tlp_ack, struct sock *sk) 421{ 422 return ret_state_update_func(sk); 423} 424""" 425 426# code replace 427is_support_tp_ca = BPF.tracepoint_exists("tcp", "tcp_cong_state_set") 428if is_support_tp_ca: 429 bpf_text = bpf_head_text + bpf_ca_tp_body_text 430else: 431 bpf_text = bpf_head_text + bpf_extra_head 432 bpf_text += bpf_no_ca_tp_body_text 433 is_support_kfunc = BPF.support_kfunc() 434 if is_support_kfunc: 435 bpf_text += kfunc_program 436 bpf_text = bpf_text.replace('SOCK_STORE_DEF', '') 437 bpf_text = bpf_text.replace('SOCK_STORE_ADD', '') 438 bpf_text = bpf_text.replace('SOCK_STORE_DEL', '') 439 else: 440 bpf_text += kprobe_program 441 bpf_text = bpf_text.replace('SOCK_STORE_DEF', 442 'BPF_HASH(sock_store, process_key_t, struct sock *);') 443 bpf_text = bpf_text.replace('SOCK_STORE_ADD', 444 'sock_store.update(&key, &sk);') 445 bpf_text = bpf_text.replace('SOCK_STORE_DEL', 446 'sock_store.delete(&key);') 447 448if args.localport: 449 bpf_text = bpf_text.replace('FILTER_LPORT', 450 'if (lport < %d || lport > %d) { return 0; }' 451 % (start_lport, end_lport)) 452else: 453 bpf_text = bpf_text.replace('FILTER_LPORT', '') 454 455if args.remoteport: 456 bpf_text = bpf_text.replace('FILTER_DPORT', 457 'if (dport < %d || dport > %d) { return 0; }' 458 % (start_rport, end_rport)) 459else: 460 bpf_text = bpf_text.replace('FILTER_DPORT', '') 461 462table_def_text = """ 463 u64 open_dura; 464 u64 loss_dura; 465 u64 disorder_dura; 466 u64 recover_dura; 467 u64 cwr_dura; 468 u64 total_changes; 469""" 470 471store_text = """ 472 datap->total_changes += 1; 473 if (last_cong_state == (TCP_CA_Open + 1)) { 474 datap->open_dura += ts; 475 } else if (last_cong_state == (TCP_CA_Disorder + 1)) { 476 datap->disorder_dura += ts; 477 } else if (last_cong_state == (TCP_CA_CWR + 1)) { 478 datap->cwr_dura += ts; 479 } else if (last_cong_state == (TCP_CA_Recovery + 1)) { 480 datap->recover_dura += ts; 481 } else if (last_cong_state == (TCP_CA_Loss + 1)) { 482 datap->loss_dura += ts; 483 } 484""" 485 486store_dist_text = """ 487 if (last_cong_state == (TCP_CA_Open + 1)) { 488 key_s.state = TCP_CA_Open; 489 } else if (last_cong_state == (TCP_CA_Disorder + 1)) { 490 key_s.state = TCP_CA_Disorder; 491 } else if (last_cong_state == (TCP_CA_CWR + 1)) { 492 key_s.state = TCP_CA_CWR; 493 } else if (last_cong_state == (TCP_CA_Recovery + 1)) { 494 key_s.state = TCP_CA_Recovery; 495 } else if (last_cong_state == (TCP_CA_Loss + 1)) { 496 key_s.state = TCP_CA_Loss; 497 } 498 TIME_UNIT 499 key_s.slot = bpf_log2l(ts); 500 dist.atomic_increment(key_s); 501""" 502 503hist_table_text = """ 504typedef struct congest_state_key { 505 u32 state; 506 u64 slot; 507}congest_state_key_t; 508 509BPF_HISTOGRAM(dist, congest_state_key_t); 510""" 511 512if args.dist: 513 bpf_text = bpf_text.replace('DEF_TEXT', '') 514 bpf_text = bpf_text.replace('STORE', store_dist_text) 515 bpf_text = bpf_text.replace('STATE_KEY', 516 'congest_state_key_t key_s = {0};') 517 bpf_text = bpf_text.replace('HIST_TABLE', hist_table_text) 518 if args.microseconds: 519 bpf_text = bpf_text.replace('TIME_UNIT', '') 520 else: 521 bpf_text = bpf_text.replace('TIME_UNIT', 'ts /= 1000;') 522else: 523 bpf_text = bpf_text.replace('DEF_TEXT', table_def_text) 524 bpf_text = bpf_text.replace('STORE', store_text) 525 bpf_text = bpf_text.replace('STATE_KEY', '') 526 bpf_text = bpf_text.replace('HIST_TABLE', '') 527 528 529if debug or args.ebpf: 530 print(bpf_text) 531 if args.ebpf: 532 exit() 533 534# load BPF program 535b = BPF(text=bpf_text) 536 537if not is_support_tp_ca and not is_support_kfunc: 538 # all the tcp congestion control status update functions 539 # are called by below 5 functions. 540 b.attach_kprobe(event="tcp_fastretrans_alert", fn_name="entry_func") 541 b.attach_kretprobe(event="tcp_fastretrans_alert", fn_name="ret_func") 542 b.attach_kprobe(event="tcp_enter_cwr", fn_name="entry_func") 543 b.attach_kretprobe(event="tcp_enter_cwr", fn_name="ret_func") 544 b.attach_kprobe(event="tcp_process_tlp_ack", fn_name="entry_func") 545 b.attach_kretprobe(event="tcp_process_tlp_ack", fn_name="ret_func") 546 b.attach_kprobe(event="tcp_enter_loss", fn_name="entry_func") 547 b.attach_kretprobe(event="tcp_enter_loss", fn_name="ret_func") 548 b.attach_kprobe(event="tcp_enter_recovery", fn_name="entry_func") 549 b.attach_kretprobe(event="tcp_enter_recovery", fn_name="ret_func") 550 551print("Tracing tcp congestion control status duration... Hit Ctrl-C to end.") 552 553 554def cong_state_to_name(state): 555 # this need to match with kernel state 556 state_name = ["open", "disorder", "cwr", "recovery", "loss"] 557 return state_name[state] 558 559# output 560exiting = 0 if args.interval else 1 561ipv6_stat = b.get_table("ipv6_stat") 562ipv4_stat = b.get_table("ipv4_stat") 563if args.dist: 564 dist = b.get_table("dist") 565label = "ms" 566if args.microseconds: 567 label = "us" 568while (1): 569 try: 570 sleep(int(args.interval)) 571 except KeyboardInterrupt: 572 exiting = 1 573 574 print() 575 if args.timestamp: 576 print("%-8s\n" % strftime("%H:%M:%S"), end="") 577 if args.dist: 578 if args.microseconds: 579 dist.print_log2_hist("usecs", "tcp_congest_state", 580 section_print_fn=cong_state_to_name) 581 else: 582 dist.print_log2_hist("msecs", "tcp_congest_state", 583 section_print_fn=cong_state_to_name) 584 dist.clear() 585 else: 586 if ipv4_stat: 587 print("%-21s% -21s %-7s %-6s %-7s %-7s %-6s %-5s" % ("LAddrPort", 588 "RAddrPort", "Open_" + label, "Dod_" + label, 589 "Rcov_" + label, "Cwr_" + label, "Los_" + label, "Chgs")) 590 laddr = "" 591 raddr = "" 592 for k, v in sorted(ipv4_stat.items(), key=lambda ipv4_stat: ipv4_stat[0].lport): 593 laddr = inet_ntop(AF_INET, pack("I", k.saddr)) 594 raddr = inet_ntop(AF_INET, pack("I", k.daddr)) 595 open_dura = v.open_dura 596 disorder_dura = v.disorder_dura 597 recover_dura = v.recover_dura 598 cwr_dura = v.cwr_dura 599 loss_dura = v.loss_dura 600 if not args.microseconds: 601 open_dura /= 1000 602 disorder_dura /= 1000 603 recover_dura /= 1000 604 cwr_dura /= 1000 605 loss_dura /= 1000 606 if v.total_changes != 0: 607 print("%-21s %-21s %-7d %-6d %-7d %-7d %-6d %-5d" % (laddr + 608 "/" + str(k.lport), raddr + "/" + str(k.dport), open_dura, 609 disorder_dura, recover_dura, cwr_dura, loss_dura, 610 v.total_changes)) 611 if ipv6_stat: 612 print("%-32s %-32s %-7s %-6s %-7s %-7s %-6s %-5s" % ("LAddrPort6", 613 "RAddrPort6", "Open_" + label, "Dod_" + label, "Rcov_" + label, 614 "Cwr_" + label, "Los_" + label, "Chgs")) 615 for k, v in sorted(ipv6_stat.items(), key=lambda ipv6_stat: ipv6_stat[0].lport): 616 laddr = inet_ntop(AF_INET6, bytes(k.saddr)) 617 raddr = inet_ntop(AF_INET6, bytes(k.daddr)) 618 open_dura = v.open_dura 619 disorder_dura = v.disorder_dura 620 recover_dura = v.recover_dura 621 cwr_dura = v.cwr_dura 622 loss_dura = v.loss_dura 623 if not args.microseconds: 624 open_dura /= 1000 625 disorder_dura /= 1000 626 recover_dura /= 1000 627 cwr_dura /= 1000 628 loss_dura /= 1000 629 if v.total_changes != 0: 630 print("%-32s %-32s %-7d %-7d %-7d %-6d %-6d %-5d" % (laddr + 631 "/" + str(k.lport), raddr + "/" + str(k.dport), open_dura, 632 disorder_dura, recover_dura, cwr_dura, loss_dura, 633 v.total_changes)) 634 ipv4_stat.clear() 635 ipv6_stat.clear() 636 countdown -= 1 637 if exiting or countdown == 0: 638 exit() 639