1*387f9dfdSAndroid Build Coastguard Worker#!/usr/bin/env python 2*387f9dfdSAndroid Build Coastguard Worker# @lint-avoid-python-3-compatibility-imports 3*387f9dfdSAndroid Build Coastguard Worker# 4*387f9dfdSAndroid Build Coastguard Worker# hardirqs Summarize hard IRQ (interrupt) event time. 5*387f9dfdSAndroid Build Coastguard Worker# For Linux, uses BCC, eBPF. 6*387f9dfdSAndroid Build Coastguard Worker# 7*387f9dfdSAndroid Build Coastguard Worker# USAGE: hardirqs [-h] [-T] [-N] [-C] [-d] [-c CPU] [interval] [outputs] 8*387f9dfdSAndroid Build Coastguard Worker# 9*387f9dfdSAndroid Build Coastguard Worker# Thanks Amer Ather for help understanding irq behavior. 10*387f9dfdSAndroid Build Coastguard Worker# 11*387f9dfdSAndroid Build Coastguard Worker# Copyright (c) 2015 Brendan Gregg. 12*387f9dfdSAndroid Build Coastguard Worker# Licensed under the Apache License, Version 2.0 (the "License") 13*387f9dfdSAndroid Build Coastguard Worker# 14*387f9dfdSAndroid Build Coastguard Worker# 19-Oct-2015 Brendan Gregg Created this. 15*387f9dfdSAndroid Build Coastguard Worker# 22-May-2021 Hengqi Chen Migrated to kernel tracepoints. 16*387f9dfdSAndroid Build Coastguard Worker# 07-Mar-2022 Rocky Xing Added CPU filter support. 17*387f9dfdSAndroid Build Coastguard Worker 18*387f9dfdSAndroid Build Coastguard Workerfrom __future__ import print_function 19*387f9dfdSAndroid Build Coastguard Workerfrom bcc import BPF 20*387f9dfdSAndroid Build Coastguard Workerfrom time import sleep, strftime 21*387f9dfdSAndroid Build Coastguard Workerimport argparse 22*387f9dfdSAndroid Build Coastguard Workerimport sys 23*387f9dfdSAndroid Build Coastguard Worker 24*387f9dfdSAndroid Build Coastguard Worker# arguments 25*387f9dfdSAndroid Build Coastguard Workerexamples = """examples: 26*387f9dfdSAndroid Build Coastguard Worker ./hardirqs # sum hard irq event time 27*387f9dfdSAndroid Build Coastguard Worker ./hardirqs -d # show hard irq event time as histograms 28*387f9dfdSAndroid Build Coastguard Worker ./hardirqs 1 10 # print 1 second summaries, 10 times 29*387f9dfdSAndroid Build Coastguard Worker ./hardirqs -NT 1 # 1s summaries, nanoseconds, and timestamps 30*387f9dfdSAndroid Build Coastguard Worker ./hardirqs -c 1 # sum hard irq event time on CPU 1 only 31*387f9dfdSAndroid Build Coastguard Worker""" 32*387f9dfdSAndroid Build Coastguard Workerparser = argparse.ArgumentParser( 33*387f9dfdSAndroid Build Coastguard Worker description="Summarize hard irq event time as histograms", 34*387f9dfdSAndroid Build Coastguard Worker formatter_class=argparse.RawDescriptionHelpFormatter, 35*387f9dfdSAndroid Build Coastguard Worker epilog=examples) 36*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-T", "--timestamp", action="store_true", 37*387f9dfdSAndroid Build Coastguard Worker help="include timestamp on output") 38*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-N", "--nanoseconds", action="store_true", 39*387f9dfdSAndroid Build Coastguard Worker help="output in nanoseconds") 40*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-C", "--count", action="store_true", 41*387f9dfdSAndroid Build Coastguard Worker help="show event counts instead of timing") 42*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-d", "--dist", action="store_true", 43*387f9dfdSAndroid Build Coastguard Worker help="show distributions as histograms") 44*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-c", "--cpu", type=int, 45*387f9dfdSAndroid Build Coastguard Worker help="trace this CPU only") 46*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("interval", nargs="?", default=99999999, 47*387f9dfdSAndroid Build Coastguard Worker help="output interval, in seconds") 48*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("outputs", nargs="?", default=99999999, 49*387f9dfdSAndroid Build Coastguard Worker help="number of outputs") 50*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("--ebpf", action="store_true", 51*387f9dfdSAndroid Build Coastguard Worker help=argparse.SUPPRESS) 52*387f9dfdSAndroid Build Coastguard Workerargs = parser.parse_args() 53*387f9dfdSAndroid Build Coastguard Workercountdown = int(args.outputs) 54*387f9dfdSAndroid Build Coastguard Workerif args.count and (args.dist or args.nanoseconds): 55*387f9dfdSAndroid Build Coastguard Worker print("The --count option can't be used with time-based options") 56*387f9dfdSAndroid Build Coastguard Worker exit() 57*387f9dfdSAndroid Build Coastguard Workerif args.count: 58*387f9dfdSAndroid Build Coastguard Worker factor = 1 59*387f9dfdSAndroid Build Coastguard Worker label = "count" 60*387f9dfdSAndroid Build Coastguard Workerelif args.nanoseconds: 61*387f9dfdSAndroid Build Coastguard Worker factor = 1 62*387f9dfdSAndroid Build Coastguard Worker label = "nsecs" 63*387f9dfdSAndroid Build Coastguard Workerelse: 64*387f9dfdSAndroid Build Coastguard Worker factor = 1000 65*387f9dfdSAndroid Build Coastguard Worker label = "usecs" 66*387f9dfdSAndroid Build Coastguard Workerdebug = 0 67*387f9dfdSAndroid Build Coastguard Worker 68*387f9dfdSAndroid Build Coastguard Worker# define BPF program 69*387f9dfdSAndroid Build Coastguard Workerbpf_text = """ 70*387f9dfdSAndroid Build Coastguard Worker#include <uapi/linux/ptrace.h> 71*387f9dfdSAndroid Build Coastguard Worker#include <linux/irq.h> 72*387f9dfdSAndroid Build Coastguard Worker#include <linux/irqdesc.h> 73*387f9dfdSAndroid Build Coastguard Worker#include <linux/interrupt.h> 74*387f9dfdSAndroid Build Coastguard Worker 75*387f9dfdSAndroid Build Coastguard Worker// Add cpu_id as part of key for irq entry event to handle the case which irq 76*387f9dfdSAndroid Build Coastguard Worker// is triggered while idle thread(swapper/x, tid=0) for each cpu core. 77*387f9dfdSAndroid Build Coastguard Worker// Please see more detail at pull request #2804, #3733. 78*387f9dfdSAndroid Build Coastguard Workertypedef struct entry_key { 79*387f9dfdSAndroid Build Coastguard Worker u32 tid; 80*387f9dfdSAndroid Build Coastguard Worker u32 cpu_id; 81*387f9dfdSAndroid Build Coastguard Worker} entry_key_t; 82*387f9dfdSAndroid Build Coastguard Worker 83*387f9dfdSAndroid Build Coastguard Workertypedef struct irq_key { 84*387f9dfdSAndroid Build Coastguard Worker char name[32]; 85*387f9dfdSAndroid Build Coastguard Worker u64 slot; 86*387f9dfdSAndroid Build Coastguard Worker} irq_key_t; 87*387f9dfdSAndroid Build Coastguard Worker 88*387f9dfdSAndroid Build Coastguard Workertypedef struct irq_name { 89*387f9dfdSAndroid Build Coastguard Worker char name[32]; 90*387f9dfdSAndroid Build Coastguard Worker} irq_name_t; 91*387f9dfdSAndroid Build Coastguard Worker 92*387f9dfdSAndroid Build Coastguard WorkerBPF_HASH(start, entry_key_t); 93*387f9dfdSAndroid Build Coastguard WorkerBPF_HASH(irqnames, entry_key_t, irq_name_t); 94*387f9dfdSAndroid Build Coastguard WorkerBPF_HISTOGRAM(dist, irq_key_t); 95*387f9dfdSAndroid Build Coastguard Worker""" 96*387f9dfdSAndroid Build Coastguard Worker 97*387f9dfdSAndroid Build Coastguard Workerbpf_text_count = """ 98*387f9dfdSAndroid Build Coastguard WorkerTRACEPOINT_PROBE(irq, irq_handler_entry) 99*387f9dfdSAndroid Build Coastguard Worker{ 100*387f9dfdSAndroid Build Coastguard Worker struct entry_key key = {}; 101*387f9dfdSAndroid Build Coastguard Worker irq_name_t name = {}; 102*387f9dfdSAndroid Build Coastguard Worker u32 cpu = bpf_get_smp_processor_id(); 103*387f9dfdSAndroid Build Coastguard Worker 104*387f9dfdSAndroid Build Coastguard Worker FILTER_CPU 105*387f9dfdSAndroid Build Coastguard Worker 106*387f9dfdSAndroid Build Coastguard Worker key.tid = bpf_get_current_pid_tgid(); 107*387f9dfdSAndroid Build Coastguard Worker key.cpu_id = cpu; 108*387f9dfdSAndroid Build Coastguard Worker 109*387f9dfdSAndroid Build Coastguard Worker TP_DATA_LOC_READ_STR(&name.name, name, sizeof(name)); 110*387f9dfdSAndroid Build Coastguard Worker irqnames.update(&key, &name); 111*387f9dfdSAndroid Build Coastguard Worker return 0; 112*387f9dfdSAndroid Build Coastguard Worker} 113*387f9dfdSAndroid Build Coastguard Worker 114*387f9dfdSAndroid Build Coastguard WorkerTRACEPOINT_PROBE(irq, irq_handler_exit) 115*387f9dfdSAndroid Build Coastguard Worker{ 116*387f9dfdSAndroid Build Coastguard Worker struct entry_key key = {}; 117*387f9dfdSAndroid Build Coastguard Worker u32 cpu = bpf_get_smp_processor_id(); 118*387f9dfdSAndroid Build Coastguard Worker 119*387f9dfdSAndroid Build Coastguard Worker FILTER_CPU 120*387f9dfdSAndroid Build Coastguard Worker 121*387f9dfdSAndroid Build Coastguard Worker key.tid = bpf_get_current_pid_tgid(); 122*387f9dfdSAndroid Build Coastguard Worker key.cpu_id = cpu; 123*387f9dfdSAndroid Build Coastguard Worker 124*387f9dfdSAndroid Build Coastguard Worker // check ret value of irq handler is not IRQ_NONE to make sure 125*387f9dfdSAndroid Build Coastguard Worker // the current event belong to this irq handler 126*387f9dfdSAndroid Build Coastguard Worker if (args->ret != IRQ_NONE) { 127*387f9dfdSAndroid Build Coastguard Worker irq_name_t *namep; 128*387f9dfdSAndroid Build Coastguard Worker 129*387f9dfdSAndroid Build Coastguard Worker namep = irqnames.lookup(&key); 130*387f9dfdSAndroid Build Coastguard Worker if (namep == 0) { 131*387f9dfdSAndroid Build Coastguard Worker return 0; // missed irq name 132*387f9dfdSAndroid Build Coastguard Worker } 133*387f9dfdSAndroid Build Coastguard Worker char *name = (char *)namep->name; 134*387f9dfdSAndroid Build Coastguard Worker irq_key_t key = {.slot = 0 /* ignore */}; 135*387f9dfdSAndroid Build Coastguard Worker 136*387f9dfdSAndroid Build Coastguard Worker bpf_probe_read_kernel(&key.name, sizeof(key.name), name); 137*387f9dfdSAndroid Build Coastguard Worker dist.atomic_increment(key); 138*387f9dfdSAndroid Build Coastguard Worker } 139*387f9dfdSAndroid Build Coastguard Worker 140*387f9dfdSAndroid Build Coastguard Worker irqnames.delete(&key); 141*387f9dfdSAndroid Build Coastguard Worker return 0; 142*387f9dfdSAndroid Build Coastguard Worker} 143*387f9dfdSAndroid Build Coastguard Worker""" 144*387f9dfdSAndroid Build Coastguard Worker 145*387f9dfdSAndroid Build Coastguard Workerbpf_text_time = """ 146*387f9dfdSAndroid Build Coastguard WorkerTRACEPOINT_PROBE(irq, irq_handler_entry) 147*387f9dfdSAndroid Build Coastguard Worker{ 148*387f9dfdSAndroid Build Coastguard Worker u64 ts = bpf_ktime_get_ns(); 149*387f9dfdSAndroid Build Coastguard Worker irq_name_t name = {}; 150*387f9dfdSAndroid Build Coastguard Worker struct entry_key key = {}; 151*387f9dfdSAndroid Build Coastguard Worker u32 cpu = bpf_get_smp_processor_id(); 152*387f9dfdSAndroid Build Coastguard Worker 153*387f9dfdSAndroid Build Coastguard Worker FILTER_CPU 154*387f9dfdSAndroid Build Coastguard Worker 155*387f9dfdSAndroid Build Coastguard Worker key.tid = bpf_get_current_pid_tgid(); 156*387f9dfdSAndroid Build Coastguard Worker key.cpu_id = cpu; 157*387f9dfdSAndroid Build Coastguard Worker 158*387f9dfdSAndroid Build Coastguard Worker TP_DATA_LOC_READ_STR(&name.name, name, sizeof(name)); 159*387f9dfdSAndroid Build Coastguard Worker irqnames.update(&key, &name); 160*387f9dfdSAndroid Build Coastguard Worker start.update(&key, &ts); 161*387f9dfdSAndroid Build Coastguard Worker return 0; 162*387f9dfdSAndroid Build Coastguard Worker} 163*387f9dfdSAndroid Build Coastguard Worker 164*387f9dfdSAndroid Build Coastguard WorkerTRACEPOINT_PROBE(irq, irq_handler_exit) 165*387f9dfdSAndroid Build Coastguard Worker{ 166*387f9dfdSAndroid Build Coastguard Worker u64 *tsp, delta; 167*387f9dfdSAndroid Build Coastguard Worker irq_name_t *namep; 168*387f9dfdSAndroid Build Coastguard Worker struct entry_key key = {}; 169*387f9dfdSAndroid Build Coastguard Worker u32 cpu = bpf_get_smp_processor_id(); 170*387f9dfdSAndroid Build Coastguard Worker 171*387f9dfdSAndroid Build Coastguard Worker key.tid = bpf_get_current_pid_tgid(); 172*387f9dfdSAndroid Build Coastguard Worker key.cpu_id = cpu; 173*387f9dfdSAndroid Build Coastguard Worker 174*387f9dfdSAndroid Build Coastguard Worker // check ret value of irq handler is not IRQ_NONE to make sure 175*387f9dfdSAndroid Build Coastguard Worker // the current event belong to this irq handler 176*387f9dfdSAndroid Build Coastguard Worker if (args->ret != IRQ_NONE) { 177*387f9dfdSAndroid Build Coastguard Worker // fetch timestamp and calculate delta 178*387f9dfdSAndroid Build Coastguard Worker tsp = start.lookup(&key); 179*387f9dfdSAndroid Build Coastguard Worker namep = irqnames.lookup(&key); 180*387f9dfdSAndroid Build Coastguard Worker if (tsp == 0 || namep == 0) { 181*387f9dfdSAndroid Build Coastguard Worker return 0; // missed start 182*387f9dfdSAndroid Build Coastguard Worker } 183*387f9dfdSAndroid Build Coastguard Worker 184*387f9dfdSAndroid Build Coastguard Worker char *name = (char *)namep->name; 185*387f9dfdSAndroid Build Coastguard Worker delta = bpf_ktime_get_ns() - *tsp; 186*387f9dfdSAndroid Build Coastguard Worker 187*387f9dfdSAndroid Build Coastguard Worker // store as sum or histogram 188*387f9dfdSAndroid Build Coastguard Worker STORE 189*387f9dfdSAndroid Build Coastguard Worker } 190*387f9dfdSAndroid Build Coastguard Worker 191*387f9dfdSAndroid Build Coastguard Worker start.delete(&key); 192*387f9dfdSAndroid Build Coastguard Worker irqnames.delete(&key); 193*387f9dfdSAndroid Build Coastguard Worker return 0; 194*387f9dfdSAndroid Build Coastguard Worker} 195*387f9dfdSAndroid Build Coastguard Worker""" 196*387f9dfdSAndroid Build Coastguard Worker 197*387f9dfdSAndroid Build Coastguard Workerif args.count: 198*387f9dfdSAndroid Build Coastguard Worker bpf_text += bpf_text_count 199*387f9dfdSAndroid Build Coastguard Workerelse: 200*387f9dfdSAndroid Build Coastguard Worker bpf_text += bpf_text_time 201*387f9dfdSAndroid Build Coastguard Worker 202*387f9dfdSAndroid Build Coastguard Worker# code substitutions 203*387f9dfdSAndroid Build Coastguard Workerif args.dist: 204*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('STORE', 205*387f9dfdSAndroid Build Coastguard Worker 'irq_key_t key = {.slot = bpf_log2l(delta / %d)};' % factor + 206*387f9dfdSAndroid Build Coastguard Worker 'bpf_probe_read_kernel(&key.name, sizeof(key.name), name);' + 207*387f9dfdSAndroid Build Coastguard Worker 'dist.atomic_increment(key);') 208*387f9dfdSAndroid Build Coastguard Workerelse: 209*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('STORE', 210*387f9dfdSAndroid Build Coastguard Worker 'irq_key_t key = {.slot = 0 /* ignore */};' + 211*387f9dfdSAndroid Build Coastguard Worker 'bpf_probe_read_kernel(&key.name, sizeof(key.name), name);' + 212*387f9dfdSAndroid Build Coastguard Worker 'dist.atomic_increment(key, delta);') 213*387f9dfdSAndroid Build Coastguard Workerif args.cpu is not None: 214*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('FILTER_CPU', 215*387f9dfdSAndroid Build Coastguard Worker 'if (cpu != %d) { return 0; }' % int(args.cpu)) 216*387f9dfdSAndroid Build Coastguard Workerelse: 217*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('FILTER_CPU', '') 218*387f9dfdSAndroid Build Coastguard Workerif debug or args.ebpf: 219*387f9dfdSAndroid Build Coastguard Worker print(bpf_text) 220*387f9dfdSAndroid Build Coastguard Worker if args.ebpf: 221*387f9dfdSAndroid Build Coastguard Worker exit() 222*387f9dfdSAndroid Build Coastguard Worker 223*387f9dfdSAndroid Build Coastguard Worker# load BPF program 224*387f9dfdSAndroid Build Coastguard Workerb = BPF(text=bpf_text) 225*387f9dfdSAndroid Build Coastguard Worker 226*387f9dfdSAndroid Build Coastguard Workerif args.count: 227*387f9dfdSAndroid Build Coastguard Worker print("Tracing hard irq events... Hit Ctrl-C to end.") 228*387f9dfdSAndroid Build Coastguard Workerelse: 229*387f9dfdSAndroid Build Coastguard Worker print("Tracing hard irq event time... Hit Ctrl-C to end.") 230*387f9dfdSAndroid Build Coastguard Worker 231*387f9dfdSAndroid Build Coastguard Worker# output 232*387f9dfdSAndroid Build Coastguard Workerexiting = 0 if args.interval else 1 233*387f9dfdSAndroid Build Coastguard Workerdist = b.get_table("dist") 234*387f9dfdSAndroid Build Coastguard Workerwhile (1): 235*387f9dfdSAndroid Build Coastguard Worker try: 236*387f9dfdSAndroid Build Coastguard Worker sleep(int(args.interval)) 237*387f9dfdSAndroid Build Coastguard Worker except KeyboardInterrupt: 238*387f9dfdSAndroid Build Coastguard Worker exiting = 1 239*387f9dfdSAndroid Build Coastguard Worker 240*387f9dfdSAndroid Build Coastguard Worker print() 241*387f9dfdSAndroid Build Coastguard Worker if args.timestamp: 242*387f9dfdSAndroid Build Coastguard Worker print("%-8s\n" % strftime("%H:%M:%S"), end="") 243*387f9dfdSAndroid Build Coastguard Worker 244*387f9dfdSAndroid Build Coastguard Worker if args.dist: 245*387f9dfdSAndroid Build Coastguard Worker dist.print_log2_hist(label, "hardirq", section_print_fn=bytes.decode) 246*387f9dfdSAndroid Build Coastguard Worker else: 247*387f9dfdSAndroid Build Coastguard Worker print("%-26s %11s" % ("HARDIRQ", "TOTAL_" + label)) 248*387f9dfdSAndroid Build Coastguard Worker for k, v in sorted(dist.items(), key=lambda dist: dist[1].value): 249*387f9dfdSAndroid Build Coastguard Worker print("%-26s %11d" % (k.name.decode('utf-8', 'replace'), v.value / factor)) 250*387f9dfdSAndroid Build Coastguard Worker dist.clear() 251*387f9dfdSAndroid Build Coastguard Worker 252*387f9dfdSAndroid Build Coastguard Worker sys.stdout.flush() 253*387f9dfdSAndroid Build Coastguard Worker 254*387f9dfdSAndroid Build Coastguard Worker countdown -= 1 255*387f9dfdSAndroid Build Coastguard Worker if exiting or countdown == 0: 256*387f9dfdSAndroid Build Coastguard Worker exit() 257