1*387f9dfdSAndroid Build Coastguard Worker#!/usr/bin/python 2*387f9dfdSAndroid Build Coastguard Worker# @lint-avoid-python-3-compatibility-imports 3*387f9dfdSAndroid Build Coastguard Worker# 4*387f9dfdSAndroid Build Coastguard Worker# softirqs Summarize soft IRQ (interrupt) event time. 5*387f9dfdSAndroid Build Coastguard Worker# For Linux, uses BCC, eBPF. 6*387f9dfdSAndroid Build Coastguard Worker# 7*387f9dfdSAndroid Build Coastguard Worker# USAGE: softirqs [-h] [-T] [-N] [-d] [interval] [count] 8*387f9dfdSAndroid Build Coastguard Worker# 9*387f9dfdSAndroid Build Coastguard Worker# Copyright (c) 2015 Brendan Gregg. 10*387f9dfdSAndroid Build Coastguard Worker# Licensed under the Apache License, Version 2.0 (the "License") 11*387f9dfdSAndroid Build Coastguard Worker# 12*387f9dfdSAndroid Build Coastguard Worker# 20-Oct-2015 Brendan Gregg Created this. 13*387f9dfdSAndroid Build Coastguard Worker 14*387f9dfdSAndroid Build Coastguard Workerfrom __future__ import print_function 15*387f9dfdSAndroid Build Coastguard Workerfrom bcc import BPF 16*387f9dfdSAndroid Build Coastguard Workerfrom time import sleep, strftime 17*387f9dfdSAndroid Build Coastguard Workerimport argparse 18*387f9dfdSAndroid Build Coastguard Worker 19*387f9dfdSAndroid Build Coastguard Worker# arguments 20*387f9dfdSAndroid Build Coastguard Workerexamples = """examples: 21*387f9dfdSAndroid Build Coastguard Worker ./softirqs # sum soft irq event time 22*387f9dfdSAndroid Build Coastguard Worker ./softirqs -d # show soft irq event time as histograms 23*387f9dfdSAndroid Build Coastguard Worker ./softirqs 1 10 # print 1 second summaries, 10 times 24*387f9dfdSAndroid Build Coastguard Worker ./softirqs -NT 1 # 1s summaries, nanoseconds, and timestamps 25*387f9dfdSAndroid Build Coastguard Worker""" 26*387f9dfdSAndroid Build Coastguard Workerparser = argparse.ArgumentParser( 27*387f9dfdSAndroid Build Coastguard Worker description="Summarize soft irq event time as histograms", 28*387f9dfdSAndroid Build Coastguard Worker formatter_class=argparse.RawDescriptionHelpFormatter, 29*387f9dfdSAndroid Build Coastguard Worker epilog=examples) 30*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-T", "--timestamp", action="store_true", 31*387f9dfdSAndroid Build Coastguard Worker help="include timestamp on output") 32*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-N", "--nanoseconds", action="store_true", 33*387f9dfdSAndroid Build Coastguard Worker help="output in nanoseconds") 34*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-d", "--dist", action="store_true", 35*387f9dfdSAndroid Build Coastguard Worker help="show distributions as histograms") 36*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-C", "--bycpu", action="store_true", 37*387f9dfdSAndroid Build Coastguard Worker help="break down softirqs to individual cpus") 38*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("interval", nargs="?", default=99999999, 39*387f9dfdSAndroid Build Coastguard Worker help="output interval, in seconds") 40*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("count", nargs="?", default=99999999, 41*387f9dfdSAndroid Build Coastguard Worker help="number of outputs") 42*387f9dfdSAndroid Build Coastguard Workerargs = parser.parse_args() 43*387f9dfdSAndroid Build Coastguard Workercountdown = int(args.count) 44*387f9dfdSAndroid Build Coastguard Workerif args.nanoseconds: 45*387f9dfdSAndroid Build Coastguard Worker factor = 1 46*387f9dfdSAndroid Build Coastguard Worker label = "nsecs" 47*387f9dfdSAndroid Build Coastguard Workerelse: 48*387f9dfdSAndroid Build Coastguard Worker factor = 1000 49*387f9dfdSAndroid Build Coastguard Worker label = "usecs" 50*387f9dfdSAndroid Build Coastguard Workerdebug = 0 51*387f9dfdSAndroid Build Coastguard Worker 52*387f9dfdSAndroid Build Coastguard Worker# define BPF program 53*387f9dfdSAndroid Build Coastguard Workerbpf_text = "" 54*387f9dfdSAndroid Build Coastguard Workerif args.bycpu: 55*387f9dfdSAndroid Build Coastguard Worker bpf_text = """ 56*387f9dfdSAndroid Build Coastguard Worker #include <uapi/linux/ptrace.h> 57*387f9dfdSAndroid Build Coastguard Worker 58*387f9dfdSAndroid Build Coastguard Worker typedef struct irq_cpu_key { 59*387f9dfdSAndroid Build Coastguard Worker s64 cpu; 60*387f9dfdSAndroid Build Coastguard Worker u64 slot; 61*387f9dfdSAndroid Build Coastguard Worker } irq_key_t; 62*387f9dfdSAndroid Build Coastguard Worker 63*387f9dfdSAndroid Build Coastguard Worker BPF_HASH(start, u32); 64*387f9dfdSAndroid Build Coastguard Worker BPF_HISTOGRAM(dist, irq_key_t); 65*387f9dfdSAndroid Build Coastguard Worker 66*387f9dfdSAndroid Build Coastguard Worker // time IRQ 67*387f9dfdSAndroid Build Coastguard Worker int trace_start_cpu(struct pt_regs *ctx) 68*387f9dfdSAndroid Build Coastguard Worker { 69*387f9dfdSAndroid Build Coastguard Worker int curr_cpu = bpf_get_smp_processor_id(); 70*387f9dfdSAndroid Build Coastguard Worker u64 ts = bpf_ktime_get_ns(); 71*387f9dfdSAndroid Build Coastguard Worker start.update(&curr_cpu, &ts); 72*387f9dfdSAndroid Build Coastguard Worker return 0; 73*387f9dfdSAndroid Build Coastguard Worker } 74*387f9dfdSAndroid Build Coastguard Worker 75*387f9dfdSAndroid Build Coastguard Worker int trace_completion_cpu(struct pt_regs *ctx) 76*387f9dfdSAndroid Build Coastguard Worker { 77*387f9dfdSAndroid Build Coastguard Worker u64 *tsp, delta; 78*387f9dfdSAndroid Build Coastguard Worker int curr_cpu = bpf_get_smp_processor_id(); 79*387f9dfdSAndroid Build Coastguard Worker 80*387f9dfdSAndroid Build Coastguard Worker // fetch timestamp and calculate delta 81*387f9dfdSAndroid Build Coastguard Worker tsp = start.lookup(&curr_cpu); 82*387f9dfdSAndroid Build Coastguard Worker COMMON 83*387f9dfdSAndroid Build Coastguard Worker 84*387f9dfdSAndroid Build Coastguard Worker // store as sum or histogram 85*387f9dfdSAndroid Build Coastguard Worker irq_key_t key = {.cpu = curr_cpu, 86*387f9dfdSAndroid Build Coastguard Worker STORE 87*387f9dfdSAndroid Build Coastguard Worker 88*387f9dfdSAndroid Build Coastguard Worker start.delete(&curr_cpu); 89*387f9dfdSAndroid Build Coastguard Worker return 0; 90*387f9dfdSAndroid Build Coastguard Worker } 91*387f9dfdSAndroid Build Coastguard Worker """ 92*387f9dfdSAndroid Build Coastguard Workerelse: 93*387f9dfdSAndroid Build Coastguard Worker bpf_text = """ 94*387f9dfdSAndroid Build Coastguard Worker #include <uapi/linux/ptrace.h> 95*387f9dfdSAndroid Build Coastguard Worker 96*387f9dfdSAndroid Build Coastguard Worker typedef struct irq_key { 97*387f9dfdSAndroid Build Coastguard Worker u64 ip; 98*387f9dfdSAndroid Build Coastguard Worker u64 slot; 99*387f9dfdSAndroid Build Coastguard Worker } irq_key_t; 100*387f9dfdSAndroid Build Coastguard Worker 101*387f9dfdSAndroid Build Coastguard Worker BPF_HASH(start, u32); 102*387f9dfdSAndroid Build Coastguard Worker BPF_HASH(iptr, u32); 103*387f9dfdSAndroid Build Coastguard Worker BPF_HISTOGRAM(dist, irq_key_t); 104*387f9dfdSAndroid Build Coastguard Worker 105*387f9dfdSAndroid Build Coastguard Worker // time IRQ 106*387f9dfdSAndroid Build Coastguard Worker int trace_start(struct pt_regs *ctx) 107*387f9dfdSAndroid Build Coastguard Worker { 108*387f9dfdSAndroid Build Coastguard Worker u32 pid = bpf_get_current_pid_tgid(); 109*387f9dfdSAndroid Build Coastguard Worker u64 ip = PT_REGS_IP(ctx), ts = bpf_ktime_get_ns(); 110*387f9dfdSAndroid Build Coastguard Worker start.update(&pid, &ts); 111*387f9dfdSAndroid Build Coastguard Worker iptr.update(&pid, &ip); 112*387f9dfdSAndroid Build Coastguard Worker return 0; 113*387f9dfdSAndroid Build Coastguard Worker } 114*387f9dfdSAndroid Build Coastguard Worker 115*387f9dfdSAndroid Build Coastguard Worker int trace_completion(struct pt_regs *ctx) 116*387f9dfdSAndroid Build Coastguard Worker { 117*387f9dfdSAndroid Build Coastguard Worker u64 *tsp, delta, ip, *ipp; 118*387f9dfdSAndroid Build Coastguard Worker u32 pid = bpf_get_current_pid_tgid(); 119*387f9dfdSAndroid Build Coastguard Worker // fetch timestamp and calculate delta 120*387f9dfdSAndroid Build Coastguard Worker tsp = start.lookup(&pid); 121*387f9dfdSAndroid Build Coastguard Worker ipp = iptr.lookup(&pid); 122*387f9dfdSAndroid Build Coastguard Worker COMMON 123*387f9dfdSAndroid Build Coastguard Worker 124*387f9dfdSAndroid Build Coastguard Worker // store as sum or histogram 125*387f9dfdSAndroid Build Coastguard Worker irq_key_t key = { 126*387f9dfdSAndroid Build Coastguard Worker STORE 127*387f9dfdSAndroid Build Coastguard Worker 128*387f9dfdSAndroid Build Coastguard Worker start.delete(&pid); 129*387f9dfdSAndroid Build Coastguard Worker iptr.delete(&pid); 130*387f9dfdSAndroid Build Coastguard Worker return 0; 131*387f9dfdSAndroid Build Coastguard Worker } 132*387f9dfdSAndroid Build Coastguard Worker """ 133*387f9dfdSAndroid Build Coastguard Worker 134*387f9dfdSAndroid Build Coastguard Worker# code substitutions 135*387f9dfdSAndroid Build Coastguard Workerbpf_text = bpf_text.replace('COMMON', 136*387f9dfdSAndroid Build Coastguard Worker """if (tsp == 0) { 137*387f9dfdSAndroid Build Coastguard Worker return 0; // missed start 138*387f9dfdSAndroid Build Coastguard Worker } 139*387f9dfdSAndroid Build Coastguard Worker delta = bpf_ktime_get_ns() - *tsp; 140*387f9dfdSAndroid Build Coastguard Worker """) 141*387f9dfdSAndroid Build Coastguard Worker 142*387f9dfdSAndroid Build Coastguard Workerif args.dist: 143*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('STORE', 144*387f9dfdSAndroid Build Coastguard Worker '.slot = bpf_log2l(delta)};' + 145*387f9dfdSAndroid Build Coastguard Worker 'dist.increment(key);') 146*387f9dfdSAndroid Build Coastguard Workerelse: 147*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('STORE', 148*387f9dfdSAndroid Build Coastguard Worker ' .ip = ip, .slot = 0 /* ignore */};' + 149*387f9dfdSAndroid Build Coastguard Worker 'u64 zero = 0, *vp = dist.lookup_or_init(&key, &zero);' + 150*387f9dfdSAndroid Build Coastguard Worker 'if (vp) { (*vp) += delta; }') 151*387f9dfdSAndroid Build Coastguard Workerif debug: 152*387f9dfdSAndroid Build Coastguard Worker print(bpf_text) 153*387f9dfdSAndroid Build Coastguard Worker 154*387f9dfdSAndroid Build Coastguard Worker# load BPF program 155*387f9dfdSAndroid Build Coastguard Workerb = BPF(text=bpf_text) 156*387f9dfdSAndroid Build Coastguard Worker 157*387f9dfdSAndroid Build Coastguard Worker# this should really use irq:softirq_entry/exit tracepoints; for now the 158*387f9dfdSAndroid Build Coastguard Worker# soft irq functions are individually traced (search your kernel for 159*387f9dfdSAndroid Build Coastguard Worker# open_softirq() calls, and adjust the following list as needed). 160*387f9dfdSAndroid Build Coastguard Workerfor softirqfunc in ("blk_iopoll_softirq", "blk_done_softirq", 161*387f9dfdSAndroid Build Coastguard Worker "rcu_process_callbacks", "run_rebalance_domains", "tasklet_action", 162*387f9dfdSAndroid Build Coastguard Worker "tasklet_hi_action", "run_timer_softirq", "net_tx_action", 163*387f9dfdSAndroid Build Coastguard Worker "net_rx_action"): 164*387f9dfdSAndroid Build Coastguard Worker if args.bycpu: 165*387f9dfdSAndroid Build Coastguard Worker b.attach_kprobe(event=softirqfunc, fn_name="trace_start_cpu") 166*387f9dfdSAndroid Build Coastguard Worker b.attach_kretprobe(event=softirqfunc, fn_name="trace_completion_cpu") 167*387f9dfdSAndroid Build Coastguard Worker else: 168*387f9dfdSAndroid Build Coastguard Worker b.attach_kprobe(event=softirqfunc, fn_name="trace_start") 169*387f9dfdSAndroid Build Coastguard Worker b.attach_kretprobe(event=softirqfunc, fn_name="trace_completion") 170*387f9dfdSAndroid Build Coastguard Worker 171*387f9dfdSAndroid Build Coastguard Workerprint("Tracing soft irq event time... Hit Ctrl-C to end.") 172*387f9dfdSAndroid Build Coastguard Worker 173*387f9dfdSAndroid Build Coastguard Worker# output 174*387f9dfdSAndroid Build Coastguard Workerexiting = 0 if args.interval else 1 175*387f9dfdSAndroid Build Coastguard Workerdist = b.get_table("dist") 176*387f9dfdSAndroid Build Coastguard Workerwhile (1): 177*387f9dfdSAndroid Build Coastguard Worker try: 178*387f9dfdSAndroid Build Coastguard Worker sleep(int(args.interval)) 179*387f9dfdSAndroid Build Coastguard Worker except KeyboardInterrupt: 180*387f9dfdSAndroid Build Coastguard Worker exiting = 1 181*387f9dfdSAndroid Build Coastguard Worker 182*387f9dfdSAndroid Build Coastguard Worker print() 183*387f9dfdSAndroid Build Coastguard Worker if args.timestamp: 184*387f9dfdSAndroid Build Coastguard Worker print("%-8s\n" % strftime("%H:%M:%S"), end="") 185*387f9dfdSAndroid Build Coastguard Worker 186*387f9dfdSAndroid Build Coastguard Worker if args.dist: 187*387f9dfdSAndroid Build Coastguard Worker if args.bycpu: 188*387f9dfdSAndroid Build Coastguard Worker dist.print_log2_hist(label, "CPU") 189*387f9dfdSAndroid Build Coastguard Worker else: 190*387f9dfdSAndroid Build Coastguard Worker dist.print_log2_hist(label, "softirq", section_print_fn=b.ksym) 191*387f9dfdSAndroid Build Coastguard Worker else: 192*387f9dfdSAndroid Build Coastguard Worker if args.bycpu: 193*387f9dfdSAndroid Build Coastguard Worker print("%-26s %11s %11s" % ("SOFTIRQ", "CPU", "TOTAL_" + label)) 194*387f9dfdSAndroid Build Coastguard Worker for k, v in sorted(dist.items(), key=lambda dist: dist[1].value): 195*387f9dfdSAndroid Build Coastguard Worker print("%-26s %11d %11d" % (b.ksym(k.ip), k.cpu, v.value / factor)) 196*387f9dfdSAndroid Build Coastguard Worker else: 197*387f9dfdSAndroid Build Coastguard Worker print("%-26s %11s" % ("SOFTIRQ", "TOTAL_" + label)) 198*387f9dfdSAndroid Build Coastguard Worker for k, v in sorted(dist.items(), key=lambda dist: dist[1].value): 199*387f9dfdSAndroid Build Coastguard Worker print("%-26s %11d" % (b.ksym(k.ip), v.value / factor)) 200*387f9dfdSAndroid Build Coastguard Worker dist.clear() 201*387f9dfdSAndroid Build Coastguard Worker 202*387f9dfdSAndroid Build Coastguard Worker countdown -= 1 203*387f9dfdSAndroid Build Coastguard Worker if exiting or countdown == 0: 204*387f9dfdSAndroid Build Coastguard Worker exit() 205