1#!/usr/bin/env python3 2 3import curses 4import operator 5import optparse 6import os 7import re 8import subprocess 9import sys 10import threading 11import queue 12 13STATS_UPDATE_INTERVAL = 0.2 14PAGE_SIZE = 4096 15 16class PagecacheStats(): 17 """Holds pagecache stats by accounting for pages added and removed. 18 19 """ 20 def __init__(self, inode_to_filename): 21 self._inode_to_filename = inode_to_filename 22 self._file_size = {} 23 self._file_pages = {} 24 self._total_pages_added = 0 25 self._total_pages_removed = 0 26 27 def add_page(self, device_number, inode, offset): 28 # See if we can find the page in our lookup table 29 if (device_number, inode) in self._inode_to_filename: 30 filename, filesize = self._inode_to_filename[(device_number, inode)] 31 if filename not in self._file_pages: 32 self._file_pages[filename] = [1, 0] 33 else: 34 self._file_pages[filename][0] += 1 35 36 self._total_pages_added += 1 37 38 if filename not in self._file_size: 39 self._file_size[filename] = filesize 40 41 def remove_page(self, device_number, inode, offset): 42 if (device_number, inode) in self._inode_to_filename: 43 filename, filesize = self._inode_to_filename[(device_number, inode)] 44 if filename not in self._file_pages: 45 self._file_pages[filename] = [0, 1] 46 else: 47 self._file_pages[filename][1] += 1 48 49 self._total_pages_removed += 1 50 51 if filename not in self._file_size: 52 self._file_size[filename] = filesize 53 54 def pages_to_mb(self, num_pages): 55 return "%.2f" % round(num_pages * PAGE_SIZE / 1024.0 / 1024.0, 2) 56 57 def bytes_to_mb(self, num_bytes): 58 return "%.2f" % round(int(num_bytes) / 1024.0 / 1024.0, 2) 59 60 def print_pages_and_mb(self, num_pages): 61 pages_string = str(num_pages) + ' (' + str(self.pages_to_mb(num_pages)) + ' MB)' 62 return pages_string 63 64 def reset_stats(self): 65 self._file_pages.clear() 66 self._total_pages_added = 0; 67 self._total_pages_removed = 0; 68 69 def print_stats(self): 70 # Create new merged dict 71 sorted_added = sorted(list(self._file_pages.items()), key=operator.itemgetter(1), reverse=True) 72 row_format = "{:<70}{:<12}{:<14}{:<9}" 73 print(row_format.format('NAME', 'ADDED (MB)', 'REMOVED (MB)', 'SIZE (MB)')) 74 for filename, added in sorted_added: 75 filesize = self._file_size[filename] 76 added = self._file_pages[filename][0] 77 removed = self._file_pages[filename][1] 78 if (len(filename) > 64): 79 filename = filename[-64:] 80 print(row_format.format(filename, self.pages_to_mb(added), self.pages_to_mb(removed), self.bytes_to_mb(filesize))) 81 82 print(row_format.format('TOTAL', self.pages_to_mb(self._total_pages_added), self.pages_to_mb(self._total_pages_removed), '')) 83 84 def print_stats_curses(self, pad): 85 sorted_added = sorted(list(self._file_pages.items()), key=operator.itemgetter(1), reverse=True) 86 height, width = pad.getmaxyx() 87 pad.clear() 88 pad.addstr(0, 2, 'NAME'.ljust(68), curses.A_REVERSE) 89 pad.addstr(0, 70, 'ADDED (MB)'.ljust(12), curses.A_REVERSE) 90 pad.addstr(0, 82, 'REMOVED (MB)'.ljust(14), curses.A_REVERSE) 91 pad.addstr(0, 96, 'SIZE (MB)'.ljust(9), curses.A_REVERSE) 92 y = 1 93 for filename, added_removed in sorted_added: 94 filesize = self._file_size[filename] 95 added = self._file_pages[filename][0] 96 removed = self._file_pages[filename][1] 97 if (len(filename) > 64): 98 filename = filename[-64:] 99 pad.addstr(y, 2, filename) 100 pad.addstr(y, 70, self.pages_to_mb(added).rjust(10)) 101 pad.addstr(y, 80, self.pages_to_mb(removed).rjust(14)) 102 pad.addstr(y, 96, self.bytes_to_mb(filesize).rjust(9)) 103 y += 1 104 if y == height - 2: 105 pad.addstr(y, 4, "<more...>") 106 break 107 y += 1 108 pad.addstr(y, 2, 'TOTAL'.ljust(74), curses.A_REVERSE) 109 pad.addstr(y, 70, str(self.pages_to_mb(self._total_pages_added)).rjust(10), curses.A_REVERSE) 110 pad.addstr(y, 80, str(self.pages_to_mb(self._total_pages_removed)).rjust(14), curses.A_REVERSE) 111 pad.refresh(0,0, 0,0, height,width) 112 113class FileReaderThread(threading.Thread): 114 """Reads data from a file/pipe on a worker thread. 115 116 Use the standard threading. Thread object API to start and interact with the 117 thread (start(), join(), etc.). 118 """ 119 120 def __init__(self, file_object, output_queue, text_file, chunk_size=-1): 121 """Initializes a FileReaderThread. 122 123 Args: 124 file_object: The file or pipe to read from. 125 output_queue: A queue.Queue object that will receive the data 126 text_file: If True, the file will be read one line at a time, and 127 chunk_size will be ignored. If False, line breaks are ignored and 128 chunk_size must be set to a positive integer. 129 chunk_size: When processing a non-text file (text_file = False), 130 chunk_size is the amount of data to copy into the queue with each 131 read operation. For text files, this parameter is ignored. 132 """ 133 threading.Thread.__init__(self) 134 self._file_object = file_object 135 self._output_queue = output_queue 136 self._text_file = text_file 137 self._chunk_size = chunk_size 138 assert text_file or chunk_size > 0 139 140 def run(self): 141 """Overrides Thread's run() function. 142 143 Returns when an EOF is encountered. 144 """ 145 if self._text_file: 146 # Read a text file one line at a time. 147 for line in self._file_object: 148 self._output_queue.put(line) 149 else: 150 # Read binary or text data until we get to EOF. 151 while True: 152 chunk = self._file_object.read(self._chunk_size) 153 if not chunk: 154 break 155 self._output_queue.put(chunk) 156 157 def set_chunk_size(self, chunk_size): 158 """Change the read chunk size. 159 160 This function can only be called if the FileReaderThread object was 161 created with an initial chunk_size > 0. 162 Args: 163 chunk_size: the new chunk size for this file. Must be > 0. 164 """ 165 # The chunk size can be changed asynchronously while a file is being read 166 # in a worker thread. However, type of file can not be changed after the 167 # the FileReaderThread has been created. These asserts verify that we are 168 # only changing the chunk size, and not the type of file. 169 assert not self._text_file 170 assert chunk_size > 0 171 self._chunk_size = chunk_size 172 173class AdbUtils(): 174 @staticmethod 175 def add_adb_serial(adb_command, device_serial): 176 if device_serial is not None: 177 adb_command.insert(1, device_serial) 178 adb_command.insert(1, '-s') 179 180 @staticmethod 181 def construct_adb_shell_command(shell_args, device_serial): 182 adb_command = ['adb', 'shell', ' '.join(shell_args)] 183 AdbUtils.add_adb_serial(adb_command, device_serial) 184 return adb_command 185 186 @staticmethod 187 def run_adb_shell(shell_args, device_serial): 188 """Runs "adb shell" with the given arguments. 189 190 Args: 191 shell_args: array of arguments to pass to adb shell. 192 device_serial: if not empty, will add the appropriate command-line 193 parameters so that adb targets the given device. 194 Returns: 195 A tuple containing the adb output (stdout & stderr) and the return code 196 from adb. Will exit if adb fails to start. 197 """ 198 adb_command = AdbUtils.construct_adb_shell_command(shell_args, device_serial) 199 200 adb_output = [] 201 adb_return_code = 0 202 try: 203 adb_output = subprocess.check_output(adb_command, stderr=subprocess.STDOUT, 204 shell=False, universal_newlines=True) 205 except OSError as error: 206 # This usually means that the adb executable was not found in the path. 207 print('\nThe command "%s" failed with the following error:' 208 % ' '.join(adb_command), file=sys.stderr) 209 print(' %s' % str(error), file=sys.stderr) 210 print('Is adb in your path?', file=sys.stderr) 211 adb_return_code = error.errno 212 adb_output = error 213 except subprocess.CalledProcessError as error: 214 # The process exited with an error. 215 adb_return_code = error.returncode 216 adb_output = error.output 217 218 return (adb_output, adb_return_code) 219 220 @staticmethod 221 def do_preprocess_adb_cmd(command, serial): 222 args = [command] 223 dump, ret_code = AdbUtils.run_adb_shell(args, serial) 224 if ret_code != 0: 225 return None 226 227 dump = ''.join(dump) 228 return dump 229 230def parse_atrace_line(line, pagecache_stats, app_name): 231 # Find a mm_filemap_add_to_page_cache entry 232 m = re.match('.* (mm_filemap_add_to_page_cache|mm_filemap_delete_from_page_cache): dev (\d+):(\d+) ino ([0-9a-z]+) page=([0-9a-z]+) pfn=([0-9a-z]+) ofs=(\d+).*', line) 233 if m != None: 234 # Get filename 235 device_number = int(m.group(2)) << 8 | int(m.group(3)) 236 if device_number == 0: 237 return 238 inode = int(m.group(4), 16) 239 if app_name != None and not (app_name in m.group(0)): 240 return 241 if m.group(1) == 'mm_filemap_add_to_page_cache': 242 pagecache_stats.add_page(device_number, inode, m.group(4)) 243 elif m.group(1) == 'mm_filemap_delete_from_page_cache': 244 pagecache_stats.remove_page(device_number, inode, m.group(4)) 245 246def build_inode_lookup_table(inode_dump): 247 inode2filename = {} 248 text = inode_dump.splitlines() 249 for line in text: 250 result = re.match('([0-9]+)d? ([0-9]+) ([0-9]+) (.*)', line) 251 if result: 252 inode2filename[(int(result.group(1)), int(result.group(2)))] = (result.group(4), result.group(3)) 253 254 return inode2filename; 255 256def get_inode_data(datafile, dumpfile, adb_serial): 257 if datafile is not None and os.path.isfile(datafile): 258 print('Using cached inode data from ' + datafile) 259 f = open(datafile, 'r') 260 stat_dump = f.read(); 261 else: 262 # Build inode maps if we were tracing page cache 263 print('Downloading inode data from device') 264 stat_dump = AdbUtils.do_preprocess_adb_cmd( 265 'find /apex /system /system_ext /product /data /vendor ' + 266 '-exec stat -c "%d %i %s %n" {} \;', adb_serial) 267 if stat_dump is None: 268 print('Could not retrieve inode data from device.') 269 sys.exit(1) 270 271 if dumpfile is not None: 272 print('Storing inode data in ' + dumpfile) 273 f = open(dumpfile, 'w') 274 f.write(stat_dump) 275 f.close() 276 277 sys.stdout.write('Done.\n') 278 279 return stat_dump 280 281def read_and_parse_trace_file(trace_file, pagecache_stats, app_name): 282 for line in trace_file: 283 parse_atrace_line(line, pagecache_stats, app_name) 284 pagecache_stats.print_stats(); 285 286def read_and_parse_trace_data_live(stdout, stderr, pagecache_stats, app_name): 287 # Start reading trace data 288 stdout_queue = queue.Queue(maxsize=128) 289 stderr_queue = queue.Queue() 290 291 stdout_thread = FileReaderThread(stdout, stdout_queue, 292 text_file=True, chunk_size=64) 293 stderr_thread = FileReaderThread(stderr, stderr_queue, 294 text_file=True) 295 stdout_thread.start() 296 stderr_thread.start() 297 298 stdscr = curses.initscr() 299 300 try: 301 height, width = stdscr.getmaxyx() 302 curses.noecho() 303 curses.cbreak() 304 stdscr.keypad(True) 305 stdscr.nodelay(True) 306 stdscr.refresh() 307 # We need at least a 30x100 window 308 used_width = max(width, 100) 309 used_height = max(height, 30) 310 311 # Create a pad for pagecache stats 312 pagecache_pad = curses.newpad(used_height - 2, used_width) 313 314 stdscr.addstr(used_height - 1, 0, 'KEY SHORTCUTS: (r)eset stats, CTRL-c to quit') 315 while (stdout_thread.isAlive() or stderr_thread.isAlive() or 316 not stdout_queue.empty() or not stderr_queue.empty()): 317 while not stderr_queue.empty(): 318 # Pass along errors from adb. 319 line = stderr_queue.get().decode("utf-8") 320 sys.stderr.write(line) 321 while True: 322 try: 323 line = stdout_queue.get(True, STATS_UPDATE_INTERVAL).decode("utf-8") 324 parse_atrace_line(line, pagecache_stats, app_name) 325 except (queue.Empty, KeyboardInterrupt): 326 break 327 328 key = '' 329 try: 330 key = stdscr.getkey() 331 except: 332 pass 333 334 if key == 'r': 335 pagecache_stats.reset_stats() 336 337 pagecache_stats.print_stats_curses(pagecache_pad) 338 except Exception as e: 339 curses.endwin() 340 print(e) 341 finally: 342 curses.endwin() 343 # The threads should already have stopped, so this is just for cleanup. 344 stdout_thread.join() 345 stderr_thread.join() 346 347 stdout.close() 348 stderr.close() 349 350def parse_options(argv): 351 usage = 'Usage: %prog [options]' 352 desc = 'Example: %prog' 353 parser = optparse.OptionParser(usage=usage, description=desc) 354 parser.add_option('-d', dest='inode_dump_file', metavar='FILE', 355 help='Dump the inode data read from a device to a file.' 356 ' This file can then be reused with the -i option to speed' 357 ' up future invocations of this script.') 358 parser.add_option('-i', dest='inode_data_file', metavar='FILE', 359 help='Read cached inode data from a file saved arlier with the' 360 ' -d option.') 361 parser.add_option('-s', '--serial', dest='device_serial', type='string', 362 help='adb device serial number') 363 parser.add_option('-f', dest='trace_file', metavar='FILE', 364 help='Show stats from a trace file, instead of running live.') 365 parser.add_option('-a', dest='app_name', type='string', 366 help='filter a particular app') 367 368 options, categories = parser.parse_args(argv[1:]) 369 if options.inode_dump_file and options.inode_data_file: 370 parser.error('options -d and -i can\'t be used at the same time') 371 return (options, categories) 372 373def main(): 374 options, categories = parse_options(sys.argv) 375 376 # Load inode data for this device 377 inode_data = get_inode_data(options.inode_data_file, options.inode_dump_file, 378 options.device_serial) 379 # Build (dev, inode) -> filename hash 380 inode_lookup_table = build_inode_lookup_table(inode_data) 381 # Init pagecache stats 382 pagecache_stats = PagecacheStats(inode_lookup_table) 383 384 if options.trace_file is not None: 385 if not os.path.isfile(options.trace_file): 386 print('Couldn\'t load trace file.', file=sys.stderr) 387 sys.exit(1) 388 trace_file = open(options.trace_file, 'r') 389 read_and_parse_trace_file(trace_file, pagecache_stats, options.app_name) 390 else: 391 # Construct and execute trace command 392 trace_cmd = AdbUtils.construct_adb_shell_command(['atrace', '--stream', 'pagecache'], 393 options.device_serial) 394 395 try: 396 atrace = subprocess.Popen(trace_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, 397 stderr=subprocess.PIPE) 398 except OSError as error: 399 print('The command failed', file=sys.stderr) 400 sys.exit(1) 401 402 read_and_parse_trace_data_live(atrace.stdout, atrace.stderr, pagecache_stats, options.app_name) 403 404if __name__ == "__main__": 405 main() 406