1#!/usr/bin/env python 2# @lint-avoid-python-3-compatibility-imports 3# 4# tcpconnect Trace TCP connect()s. 5# For Linux, uses BCC, eBPF. Embedded C. 6# 7# USAGE: tcpconnect [-h] [-c] [-t] [-p PID] [-P PORT [PORT ...]] [-4 | -6] 8# 9# All connection attempts are traced, even if they ultimately fail. 10# 11# This uses dynamic tracing of kernel functions, and will need to be updated 12# to match kernel changes. 13# 14# Copyright (c) 2015 Brendan Gregg. 15# Licensed under the Apache License, Version 2.0 (the "License") 16# 17# 25-Sep-2015 Brendan Gregg Created this. 18# 14-Feb-2016 " " Switch to bpf_perf_output. 19# 09-Jan-2019 Takuma Kume Support filtering by UID 20# 30-Jul-2019 Xiaozhou Liu Count connects. 21# 07-Oct-2020 Nabil Schear Correlate connects with DNS responses 22# 08-Mar-2021 Suresh Kumar Added LPORT option 23 24from __future__ import print_function 25from bcc import BPF 26from bcc.containers import filter_by_containers 27from bcc.utils import printb 28import argparse 29from socket import inet_ntop, ntohs, AF_INET, AF_INET6 30from struct import pack 31from time import sleep 32from datetime import datetime 33 34# arguments 35examples = """examples: 36 ./tcpconnect # trace all TCP connect()s 37 ./tcpconnect -t # include timestamps 38 ./tcpconnect -d # include DNS queries associated with connects 39 ./tcpconnect -p 181 # only trace PID 181 40 ./tcpconnect -P 80 # only trace port 80 41 ./tcpconnect -P 80,81 # only trace port 80 and 81 42 ./tcpconnect -4 # only trace IPv4 family 43 ./tcpconnect -6 # only trace IPv6 family 44 ./tcpconnect -U # include UID 45 ./tcpconnect -u 1000 # only trace UID 1000 46 ./tcpconnect -c # count connects per src ip and dest ip/port 47 ./tcpconnect -L # include LPORT while printing outputs 48 ./tcpconnect --cgroupmap mappath # only trace cgroups in this BPF map 49 ./tcpconnect --mntnsmap mappath # only trace mount namespaces in the map 50""" 51parser = argparse.ArgumentParser( 52 description="Trace TCP connects", 53 formatter_class=argparse.RawDescriptionHelpFormatter, 54 epilog=examples) 55parser.add_argument("-t", "--timestamp", action="store_true", 56 help="include timestamp on output") 57parser.add_argument("-p", "--pid", 58 help="trace this PID only") 59parser.add_argument("-P", "--port", 60 help="comma-separated list of destination ports to trace.") 61group = parser.add_mutually_exclusive_group() 62group.add_argument("-4", "--ipv4", action="store_true", 63 help="trace IPv4 family only") 64group.add_argument("-6", "--ipv6", action="store_true", 65 help="trace IPv6 family only") 66parser.add_argument("-L", "--lport", action="store_true", 67 help="include LPORT on output") 68parser.add_argument("-U", "--print-uid", action="store_true", 69 help="include UID on output") 70parser.add_argument("-u", "--uid", 71 help="trace this UID only") 72parser.add_argument("-c", "--count", action="store_true", 73 help="count connects per src ip and dest ip/port") 74parser.add_argument("--cgroupmap", 75 help="trace cgroups in this BPF map only") 76parser.add_argument("--mntnsmap", 77 help="trace mount namespaces in this BPF map only") 78parser.add_argument("-d", "--dns", action="store_true", 79 help="include likely DNS query associated with each connect") 80parser.add_argument("--ebpf", action="store_true", 81 help=argparse.SUPPRESS) 82args = parser.parse_args() 83debug = 0 84 85# define BPF program 86bpf_text = """ 87#include <uapi/linux/ptrace.h> 88#include <net/sock.h> 89#include <bcc/proto.h> 90 91BPF_HASH(currsock, u32, struct sock *); 92 93// separate data structs for ipv4 and ipv6 94struct ipv4_data_t { 95 u64 ts_us; 96 u32 pid; 97 u32 uid; 98 u32 saddr; 99 u32 daddr; 100 u64 ip; 101 u16 lport; 102 u16 dport; 103 char task[TASK_COMM_LEN]; 104}; 105BPF_PERF_OUTPUT(ipv4_events); 106 107struct ipv6_data_t { 108 u64 ts_us; 109 u32 pid; 110 u32 uid; 111 unsigned __int128 saddr; 112 unsigned __int128 daddr; 113 u64 ip; 114 u16 lport; 115 u16 dport; 116 char task[TASK_COMM_LEN]; 117}; 118BPF_PERF_OUTPUT(ipv6_events); 119 120// separate flow keys per address family 121struct ipv4_flow_key_t { 122 u32 saddr; 123 u32 daddr; 124 u16 dport; 125}; 126BPF_HASH(ipv4_count, struct ipv4_flow_key_t); 127 128struct ipv6_flow_key_t { 129 unsigned __int128 saddr; 130 unsigned __int128 daddr; 131 u16 dport; 132}; 133BPF_HASH(ipv6_count, struct ipv6_flow_key_t); 134 135int trace_connect_entry(struct pt_regs *ctx, struct sock *sk) 136{ 137 if (container_should_be_filtered()) { 138 return 0; 139 } 140 141 u64 pid_tgid = bpf_get_current_pid_tgid(); 142 u32 pid = pid_tgid >> 32; 143 u32 tid = pid_tgid; 144 FILTER_PID 145 146 u32 uid = bpf_get_current_uid_gid(); 147 FILTER_UID 148 149 // stash the sock ptr for lookup on return 150 currsock.update(&tid, &sk); 151 152 return 0; 153}; 154 155static int trace_connect_return(struct pt_regs *ctx, short ipver) 156{ 157 int ret = PT_REGS_RC(ctx); 158 u64 pid_tgid = bpf_get_current_pid_tgid(); 159 u32 pid = pid_tgid >> 32; 160 u32 tid = pid_tgid; 161 162 struct sock **skpp; 163 skpp = currsock.lookup(&tid); 164 if (skpp == 0) { 165 return 0; // missed entry 166 } 167 168 if (ret != 0) { 169 // failed to send SYNC packet, may not have populated 170 // socket __sk_common.{skc_rcv_saddr, ...} 171 currsock.delete(&tid); 172 return 0; 173 } 174 175 // pull in details 176 struct sock *skp = *skpp; 177 u16 lport = skp->__sk_common.skc_num; 178 u16 dport = skp->__sk_common.skc_dport; 179 180 FILTER_PORT 181 182 FILTER_FAMILY 183 184 if (ipver == 4) { 185 IPV4_CODE 186 } else /* 6 */ { 187 IPV6_CODE 188 } 189 190 currsock.delete(&tid); 191 192 return 0; 193} 194 195int trace_connect_v4_return(struct pt_regs *ctx) 196{ 197 return trace_connect_return(ctx, 4); 198} 199 200int trace_connect_v6_return(struct pt_regs *ctx) 201{ 202 return trace_connect_return(ctx, 6); 203} 204""" 205 206struct_init = {'ipv4': 207 {'count': 208 """ 209 struct ipv4_flow_key_t flow_key = {}; 210 flow_key.saddr = skp->__sk_common.skc_rcv_saddr; 211 flow_key.daddr = skp->__sk_common.skc_daddr; 212 flow_key.dport = ntohs(dport); 213 ipv4_count.increment(flow_key);""", 214 'trace': 215 """ 216 struct ipv4_data_t data4 = {.pid = pid, .ip = ipver}; 217 data4.uid = bpf_get_current_uid_gid(); 218 data4.ts_us = bpf_ktime_get_ns() / 1000; 219 data4.saddr = skp->__sk_common.skc_rcv_saddr; 220 data4.daddr = skp->__sk_common.skc_daddr; 221 data4.lport = lport; 222 data4.dport = ntohs(dport); 223 bpf_get_current_comm(&data4.task, sizeof(data4.task)); 224 ipv4_events.perf_submit(ctx, &data4, sizeof(data4));""" 225 }, 226 'ipv6': 227 {'count': 228 """ 229 struct ipv6_flow_key_t flow_key = {}; 230 bpf_probe_read_kernel(&flow_key.saddr, sizeof(flow_key.saddr), 231 skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); 232 bpf_probe_read_kernel(&flow_key.daddr, sizeof(flow_key.daddr), 233 skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32); 234 flow_key.dport = ntohs(dport); 235 ipv6_count.increment(flow_key);""", 236 'trace': 237 """ 238 struct ipv6_data_t data6 = {.pid = pid, .ip = ipver}; 239 data6.uid = bpf_get_current_uid_gid(); 240 data6.ts_us = bpf_ktime_get_ns() / 1000; 241 bpf_probe_read_kernel(&data6.saddr, sizeof(data6.saddr), 242 skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); 243 bpf_probe_read_kernel(&data6.daddr, sizeof(data6.daddr), 244 skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32); 245 data6.lport = lport; 246 data6.dport = ntohs(dport); 247 bpf_get_current_comm(&data6.task, sizeof(data6.task)); 248 ipv6_events.perf_submit(ctx, &data6, sizeof(data6));""" 249 } 250 } 251 252# This defines an additional BPF program that instruments udp_recvmsg system 253# call to locate DNS response packets on UDP port 53. When these packets are 254# located, the data is copied to user-space where python will parse them with 255# dnslib. 256# 257# uses a percpu array of length 1 to store the dns_data_t off the stack to 258# allow for a maximum DNS packet length of 512 bytes. 259dns_bpf_text = """ 260#include <net/inet_sock.h> 261 262#define MAX_PKT 512 263struct dns_data_t { 264 u8 pkt[MAX_PKT]; 265}; 266 267BPF_PERF_OUTPUT(dns_events); 268 269// store msghdr pointer captured on syscall entry to parse on syscall return 270BPF_HASH(tbl_udp_msg_hdr, u64, struct msghdr *); 271 272// single element per-cpu array to hold the current event off the stack 273BPF_PERCPU_ARRAY(dns_data,struct dns_data_t,1); 274 275int trace_udp_recvmsg(struct pt_regs *ctx) 276{ 277 __u64 pid_tgid = bpf_get_current_pid_tgid(); 278 struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx); 279 struct inet_sock *is = inet_sk(sk); 280 281 // only grab port 53 packets, 13568 is ntohs(53) 282 if (is->inet_dport == 13568) { 283 struct msghdr *msghdr = (struct msghdr *)PT_REGS_PARM2(ctx); 284 tbl_udp_msg_hdr.update(&pid_tgid, &msghdr); 285 } 286 return 0; 287} 288 289int trace_udp_ret_recvmsg(struct pt_regs *ctx) 290{ 291 __u64 pid_tgid = bpf_get_current_pid_tgid(); 292 u32 zero = 0; 293 struct msghdr **msgpp = tbl_udp_msg_hdr.lookup(&pid_tgid); 294 if (msgpp == 0) 295 return 0; 296 297 struct msghdr *msghdr = (struct msghdr *)*msgpp; 298 if (msghdr->msg_iter.TYPE_FIELD != ITER_IOVEC) 299 goto delete_and_return; 300 301 int copied = (int)PT_REGS_RC(ctx); 302 if (copied < 0) 303 goto delete_and_return; 304 size_t buflen = (size_t)copied; 305 306 if (buflen > msghdr->msg_iter.iov->iov_len) 307 goto delete_and_return; 308 309 if (buflen > MAX_PKT) 310 buflen = MAX_PKT; 311 312 struct dns_data_t *data = dns_data.lookup(&zero); 313 if (!data) // this should never happen, just making the verifier happy 314 return 0; 315 316 void *iovbase = msghdr->msg_iter.iov->iov_base; 317 bpf_probe_read(data->pkt, buflen, iovbase); 318 dns_events.perf_submit(ctx, data, buflen); 319 320delete_and_return: 321 tbl_udp_msg_hdr.delete(&pid_tgid); 322 return 0; 323} 324 325#include <uapi/linux/udp.h> 326 327int trace_udpv6_recvmsg(struct pt_regs *ctx) 328{ 329 struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM2(ctx); 330 struct udphdr *hdr = (void*)skb->head + skb->transport_header; 331 struct dns_data_t *event; 332 int zero = 0; 333 void *data; 334 335 /* hex(53) = 0x0035, htons(0x0035) = 0x3500 */ 336 if (hdr->source != 0x3500) 337 return 0; 338 339 /* skip UDP header */ 340 data = skb->data + 8; 341 342 event = dns_data.lookup(&zero); 343 if (!event) 344 return 0; 345 346 bpf_probe_read(event->pkt, sizeof(event->pkt), data); 347 dns_events.perf_submit(ctx, event, sizeof(*event)); 348 return 0; 349} 350""" 351 352if args.count and args.dns: 353 print("Error: you may not specify -d/--dns with -c/--count.") 354 exit() 355 356# code substitutions 357if args.count: 358 bpf_text = bpf_text.replace("IPV4_CODE", struct_init['ipv4']['count']) 359 bpf_text = bpf_text.replace("IPV6_CODE", struct_init['ipv6']['count']) 360else: 361 bpf_text = bpf_text.replace("IPV4_CODE", struct_init['ipv4']['trace']) 362 bpf_text = bpf_text.replace("IPV6_CODE", struct_init['ipv6']['trace']) 363 364if args.pid: 365 bpf_text = bpf_text.replace('FILTER_PID', 366 'if (pid != %s) { return 0; }' % args.pid) 367if args.port: 368 dports = [int(dport) for dport in args.port.split(',')] 369 dports_if = ' && '.join(['dport != %d' % ntohs(dport) for dport in dports]) 370 bpf_text = bpf_text.replace('FILTER_PORT', 371 'if (%s) { currsock.delete(&tid); return 0; }' % dports_if) 372if args.ipv4: 373 bpf_text = bpf_text.replace('FILTER_FAMILY', 374 'if (ipver != 4) { return 0; }') 375elif args.ipv6: 376 bpf_text = bpf_text.replace('FILTER_FAMILY', 377 'if (ipver != 6) { return 0; }') 378if args.uid: 379 bpf_text = bpf_text.replace('FILTER_UID', 380 'if (uid != %s) { return 0; }' % args.uid) 381bpf_text = filter_by_containers(args) + bpf_text 382 383bpf_text = bpf_text.replace('FILTER_PID', '') 384bpf_text = bpf_text.replace('FILTER_PORT', '') 385bpf_text = bpf_text.replace('FILTER_FAMILY', '') 386bpf_text = bpf_text.replace('FILTER_UID', '') 387 388if args.dns: 389 if BPF.kernel_struct_has_field(b'iov_iter', b'iter_type') == 1: 390 dns_bpf_text = dns_bpf_text.replace('TYPE_FIELD', 'iter_type') 391 else: 392 dns_bpf_text = dns_bpf_text.replace('TYPE_FIELD', 'type') 393 bpf_text += dns_bpf_text 394 395if debug or args.ebpf: 396 print(bpf_text) 397 if args.ebpf: 398 exit() 399 400# process event 401def print_ipv4_event(cpu, data, size): 402 event = b["ipv4_events"].event(data) 403 global start_ts 404 if args.timestamp: 405 if start_ts == 0: 406 start_ts = event.ts_us 407 printb(b"%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), nl="") 408 if args.print_uid: 409 printb(b"%-6d" % event.uid, nl="") 410 dest_ip = inet_ntop(AF_INET, pack("I", event.daddr)).encode() 411 if args.lport: 412 printb(b"%-7d %-12.12s %-2d %-16s %-6d %-16s %-6d %s" % (event.pid, 413 event.task, event.ip, 414 inet_ntop(AF_INET, pack("I", event.saddr)).encode(), event.lport, 415 dest_ip, event.dport, print_dns(dest_ip))) 416 else: 417 printb(b"%-7d %-12.12s %-2d %-16s %-16s %-6d %s" % (event.pid, 418 event.task, event.ip, 419 inet_ntop(AF_INET, pack("I", event.saddr)).encode(), 420 dest_ip, event.dport, print_dns(dest_ip))) 421 422def print_ipv6_event(cpu, data, size): 423 event = b["ipv6_events"].event(data) 424 global start_ts 425 if args.timestamp: 426 if start_ts == 0: 427 start_ts = event.ts_us 428 printb(b"%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), nl="") 429 if args.print_uid: 430 printb(b"%-6d" % event.uid, nl="") 431 dest_ip = inet_ntop(AF_INET6, event.daddr).encode() 432 if args.lport: 433 printb(b"%-7d %-12.12s %-2d %-16s %-6d %-16s %-6d %s" % (event.pid, 434 event.task, event.ip, 435 inet_ntop(AF_INET6, event.saddr).encode(), event.lport, 436 dest_ip, event.dport, print_dns(dest_ip))) 437 else: 438 printb(b"%-7d %-12.12s %-2d %-16s %-16s %-6d %s" % (event.pid, 439 event.task, event.ip, 440 inet_ntop(AF_INET6, event.saddr).encode(), 441 dest_ip, event.dport, print_dns(dest_ip))) 442 443def depict_cnt(counts_tab, l3prot='ipv4'): 444 for k, v in sorted(counts_tab.items(), 445 key=lambda counts: counts[1].value, reverse=True): 446 depict_key = "" 447 if l3prot == 'ipv4': 448 depict_key = "%-25s %-25s %-20s" % \ 449 ((inet_ntop(AF_INET, pack('I', k.saddr))), 450 inet_ntop(AF_INET, pack('I', k.daddr)), k.dport) 451 else: 452 depict_key = "%-25s %-25s %-20s" % \ 453 ((inet_ntop(AF_INET6, k.saddr)), 454 inet_ntop(AF_INET6, k.daddr), k.dport) 455 456 print("%s %-10d" % (depict_key, v.value)) 457 458def print_dns(dest_ip): 459 if not args.dns: 460 return b"" 461 462 dnsname, timestamp = dns_cache.get(dest_ip, (None, None)) 463 if timestamp is not None: 464 diff = datetime.now() - timestamp 465 diff = float(diff.seconds) * 1000 + float(diff.microseconds) / 1000 466 else: 467 diff = 0 468 if dnsname is None: 469 dnsname = b"No DNS Query" 470 if dest_ip == b"127.0.0.1" or dest_ip == b"::1": 471 dnsname = b"localhost" 472 retval = b"%s" % dnsname 473 if diff > DELAY_DNS: 474 retval += b" (%.3fms)" % diff 475 return retval 476 477if args.dns: 478 try: 479 import dnslib 480 from cachetools import TTLCache 481 except ImportError: 482 print("Error: The python packages dnslib and cachetools are required " 483 "to use the -d/--dns option.") 484 print("Install this package with:") 485 print("\t$ pip3 install dnslib cachetools") 486 print(" or") 487 print("\t$ sudo apt-get install python3-dnslib python3-cachetools " 488 "(on Ubuntu 18.04+)") 489 exit(1) 490 491 # 24 hours 492 DEFAULT_TTL = 86400 493 494 # Cache Size in entries 495 DNS_CACHE_SIZE = 10240 496 497 # delay in ms in which to warn users of long delay between the query 498 # and the connect that used the IP 499 DELAY_DNS = 100 500 501 dns_cache = TTLCache(maxsize=DNS_CACHE_SIZE, ttl=DEFAULT_TTL) 502 503 # process event 504 def save_dns(cpu, data, size): 505 event = b["dns_events"].event(data) 506 payload = event.pkt[:size] 507 508 # pass the payload to dnslib for parsing 509 dnspkt = dnslib.DNSRecord.parse(payload) 510 # lets only look at responses 511 if dnspkt.header.qr != 1: 512 return 513 # must be some questions in there 514 if dnspkt.header.q != 1: 515 return 516 # make sure there are answers 517 if dnspkt.header.a == 0 and dnspkt.header.aa == 0: 518 return 519 520 # lop off the trailing . 521 question = ("%s" % dnspkt.q.qname)[:-1].encode('utf-8') 522 523 for answer in dnspkt.rr: 524 # skip all but A and AAAA records 525 if answer.rtype == 1 or answer.rtype == 28: 526 dns_cache[str(answer.rdata).encode('utf-8')] = (question, 527 datetime.now()) 528 529# initialize BPF 530b = BPF(text=bpf_text) 531b.attach_kprobe(event="tcp_v4_connect", fn_name="trace_connect_entry") 532b.attach_kprobe(event="tcp_v6_connect", fn_name="trace_connect_entry") 533b.attach_kretprobe(event="tcp_v4_connect", fn_name="trace_connect_v4_return") 534b.attach_kretprobe(event="tcp_v6_connect", fn_name="trace_connect_v6_return") 535if args.dns: 536 b.attach_kprobe(event="udp_recvmsg", fn_name="trace_udp_recvmsg") 537 b.attach_kretprobe(event="udp_recvmsg", fn_name="trace_udp_ret_recvmsg") 538 b.attach_kprobe(event="udpv6_queue_rcv_one_skb", fn_name="trace_udpv6_recvmsg") 539 540print("Tracing connect ... Hit Ctrl-C to end") 541if args.count: 542 try: 543 while True: 544 sleep(99999999) 545 except KeyboardInterrupt: 546 pass 547 548 # header 549 print("\n%-25s %-25s %-20s %-10s" % ( 550 "LADDR", "RADDR", "RPORT", "CONNECTS")) 551 depict_cnt(b["ipv4_count"]) 552 depict_cnt(b["ipv6_count"], l3prot='ipv6') 553# read events 554else: 555 # header 556 if args.timestamp: 557 print("%-9s" % ("TIME(s)"), end="") 558 if args.print_uid: 559 print("%-6s" % ("UID"), end="") 560 if args.lport: 561 print("%-7s %-12s %-2s %-16s %-6s %-16s %-6s" % ("PID", "COMM", "IP", "SADDR", 562 "LPORT", "DADDR", "DPORT"), end="") 563 else: 564 print("%-7s %-12s %-2s %-16s %-16s %-6s" % ("PID", "COMM", "IP", "SADDR", 565 "DADDR", "DPORT"), end="") 566 if args.dns: 567 print(" QUERY") 568 else: 569 print() 570 571 start_ts = 0 572 573 # read events 574 b["ipv4_events"].open_perf_buffer(print_ipv4_event) 575 b["ipv6_events"].open_perf_buffer(print_ipv6_event) 576 if args.dns: 577 b["dns_events"].open_perf_buffer(save_dns) 578 while True: 579 try: 580 b.perf_buffer_poll() 581 except KeyboardInterrupt: 582 exit() 583