xref: /aosp_15_r20/system/extras/pagecache/pagecache.py (revision 288bf5226967eb3dac5cce6c939ccc2a7f2b4fe5)
1#!/usr/bin/env python3
2
3import curses
4import operator
5import optparse
6import os
7import re
8import subprocess
9import sys
10import threading
11import queue
12
13STATS_UPDATE_INTERVAL = 0.2
14PAGE_SIZE = 4096
15
16class PagecacheStats():
17  """Holds pagecache stats by accounting for pages added and removed.
18
19  """
20  def __init__(self, inode_to_filename):
21    self._inode_to_filename = inode_to_filename
22    self._file_size = {}
23    self._file_pages = {}
24    self._total_pages_added = 0
25    self._total_pages_removed = 0
26
27  def add_page(self, device_number, inode, offset):
28    # See if we can find the page in our lookup table
29    if (device_number, inode) in self._inode_to_filename:
30      filename, filesize = self._inode_to_filename[(device_number, inode)]
31      if filename not in self._file_pages:
32        self._file_pages[filename] = [1, 0]
33      else:
34        self._file_pages[filename][0] += 1
35
36      self._total_pages_added += 1
37
38      if filename not in self._file_size:
39        self._file_size[filename] = filesize
40
41  def remove_page(self, device_number, inode, offset):
42    if (device_number, inode) in self._inode_to_filename:
43      filename, filesize = self._inode_to_filename[(device_number, inode)]
44      if filename not in self._file_pages:
45        self._file_pages[filename] = [0, 1]
46      else:
47        self._file_pages[filename][1] += 1
48
49      self._total_pages_removed += 1
50
51      if filename not in self._file_size:
52        self._file_size[filename] = filesize
53
54  def pages_to_mb(self, num_pages):
55    return "%.2f" % round(num_pages * PAGE_SIZE / 1024.0 / 1024.0, 2)
56
57  def bytes_to_mb(self, num_bytes):
58    return "%.2f" % round(int(num_bytes) / 1024.0 / 1024.0, 2)
59
60  def print_pages_and_mb(self, num_pages):
61    pages_string = str(num_pages) + ' (' + str(self.pages_to_mb(num_pages)) + ' MB)'
62    return pages_string
63
64  def reset_stats(self):
65    self._file_pages.clear()
66    self._total_pages_added = 0;
67    self._total_pages_removed = 0;
68
69  def print_stats(self):
70    # Create new merged dict
71    sorted_added = sorted(list(self._file_pages.items()), key=operator.itemgetter(1), reverse=True)
72    row_format = "{:<70}{:<12}{:<14}{:<9}"
73    print(row_format.format('NAME', 'ADDED (MB)', 'REMOVED (MB)', 'SIZE (MB)'))
74    for filename, added in sorted_added:
75      filesize = self._file_size[filename]
76      added = self._file_pages[filename][0]
77      removed = self._file_pages[filename][1]
78      if (len(filename) > 64):
79        filename = filename[-64:]
80      print(row_format.format(filename, self.pages_to_mb(added), self.pages_to_mb(removed), self.bytes_to_mb(filesize)))
81
82    print(row_format.format('TOTAL', self.pages_to_mb(self._total_pages_added), self.pages_to_mb(self._total_pages_removed), ''))
83
84  def print_stats_curses(self, pad):
85    sorted_added = sorted(list(self._file_pages.items()), key=operator.itemgetter(1), reverse=True)
86    height, width = pad.getmaxyx()
87    pad.clear()
88    pad.addstr(0, 2, 'NAME'.ljust(68), curses.A_REVERSE)
89    pad.addstr(0, 70, 'ADDED (MB)'.ljust(12), curses.A_REVERSE)
90    pad.addstr(0, 82, 'REMOVED (MB)'.ljust(14), curses.A_REVERSE)
91    pad.addstr(0, 96, 'SIZE (MB)'.ljust(9), curses.A_REVERSE)
92    y = 1
93    for filename, added_removed in sorted_added:
94      filesize = self._file_size[filename]
95      added  = self._file_pages[filename][0]
96      removed = self._file_pages[filename][1]
97      if (len(filename) > 64):
98        filename = filename[-64:]
99      pad.addstr(y, 2, filename)
100      pad.addstr(y, 70, self.pages_to_mb(added).rjust(10))
101      pad.addstr(y, 80, self.pages_to_mb(removed).rjust(14))
102      pad.addstr(y, 96, self.bytes_to_mb(filesize).rjust(9))
103      y += 1
104      if y == height - 2:
105        pad.addstr(y, 4, "<more...>")
106        break
107    y += 1
108    pad.addstr(y, 2, 'TOTAL'.ljust(74), curses.A_REVERSE)
109    pad.addstr(y, 70, str(self.pages_to_mb(self._total_pages_added)).rjust(10), curses.A_REVERSE)
110    pad.addstr(y, 80, str(self.pages_to_mb(self._total_pages_removed)).rjust(14), curses.A_REVERSE)
111    pad.refresh(0,0, 0,0, height,width)
112
113class FileReaderThread(threading.Thread):
114  """Reads data from a file/pipe on a worker thread.
115
116  Use the standard threading. Thread object API to start and interact with the
117  thread (start(), join(), etc.).
118  """
119
120  def __init__(self, file_object, output_queue, text_file, chunk_size=-1):
121    """Initializes a FileReaderThread.
122
123    Args:
124      file_object: The file or pipe to read from.
125      output_queue: A queue.Queue object that will receive the data
126      text_file: If True, the file will be read one line at a time, and
127          chunk_size will be ignored.  If False, line breaks are ignored and
128          chunk_size must be set to a positive integer.
129      chunk_size: When processing a non-text file (text_file = False),
130          chunk_size is the amount of data to copy into the queue with each
131          read operation.  For text files, this parameter is ignored.
132    """
133    threading.Thread.__init__(self)
134    self._file_object = file_object
135    self._output_queue = output_queue
136    self._text_file = text_file
137    self._chunk_size = chunk_size
138    assert text_file or chunk_size > 0
139
140  def run(self):
141    """Overrides Thread's run() function.
142
143    Returns when an EOF is encountered.
144    """
145    if self._text_file:
146      # Read a text file one line at a time.
147      for line in self._file_object:
148        self._output_queue.put(line)
149    else:
150      # Read binary or text data until we get to EOF.
151      while True:
152        chunk = self._file_object.read(self._chunk_size)
153        if not chunk:
154          break
155        self._output_queue.put(chunk)
156
157  def set_chunk_size(self, chunk_size):
158    """Change the read chunk size.
159
160    This function can only be called if the FileReaderThread object was
161    created with an initial chunk_size > 0.
162    Args:
163      chunk_size: the new chunk size for this file.  Must be > 0.
164    """
165    # The chunk size can be changed asynchronously while a file is being read
166    # in a worker thread.  However, type of file can not be changed after the
167    # the FileReaderThread has been created.  These asserts verify that we are
168    # only changing the chunk size, and not the type of file.
169    assert not self._text_file
170    assert chunk_size > 0
171    self._chunk_size = chunk_size
172
173class AdbUtils():
174  @staticmethod
175  def add_adb_serial(adb_command, device_serial):
176    if device_serial is not None:
177      adb_command.insert(1, device_serial)
178      adb_command.insert(1, '-s')
179
180  @staticmethod
181  def construct_adb_shell_command(shell_args, device_serial):
182    adb_command = ['adb', 'shell', ' '.join(shell_args)]
183    AdbUtils.add_adb_serial(adb_command, device_serial)
184    return adb_command
185
186  @staticmethod
187  def run_adb_shell(shell_args, device_serial):
188    """Runs "adb shell" with the given arguments.
189
190    Args:
191      shell_args: array of arguments to pass to adb shell.
192      device_serial: if not empty, will add the appropriate command-line
193          parameters so that adb targets the given device.
194    Returns:
195      A tuple containing the adb output (stdout & stderr) and the return code
196      from adb.  Will exit if adb fails to start.
197    """
198    adb_command = AdbUtils.construct_adb_shell_command(shell_args, device_serial)
199
200    adb_output = []
201    adb_return_code = 0
202    try:
203      adb_output = subprocess.check_output(adb_command, stderr=subprocess.STDOUT,
204                                           shell=False, universal_newlines=True)
205    except OSError as error:
206      # This usually means that the adb executable was not found in the path.
207      print('\nThe command "%s" failed with the following error:'
208                            % ' '.join(adb_command), file=sys.stderr)
209      print('    %s' % str(error), file=sys.stderr)
210      print('Is adb in your path?', file=sys.stderr)
211      adb_return_code = error.errno
212      adb_output = error
213    except subprocess.CalledProcessError as error:
214      # The process exited with an error.
215      adb_return_code = error.returncode
216      adb_output = error.output
217
218    return (adb_output, adb_return_code)
219
220  @staticmethod
221  def do_preprocess_adb_cmd(command, serial):
222    args = [command]
223    dump, ret_code = AdbUtils.run_adb_shell(args, serial)
224    if ret_code != 0:
225      return None
226
227    dump = ''.join(dump)
228    return dump
229
230def parse_atrace_line(line, pagecache_stats, app_name):
231  # Find a mm_filemap_add_to_page_cache entry
232  m = re.match('.* (mm_filemap_add_to_page_cache|mm_filemap_delete_from_page_cache): dev (\d+):(\d+) ino ([0-9a-z]+) page=([0-9a-z]+) pfn=([0-9a-z]+) ofs=(\d+).*', line)
233  if m != None:
234    # Get filename
235    device_number = int(m.group(2)) << 8 | int(m.group(3))
236    if device_number == 0:
237      return
238    inode = int(m.group(4), 16)
239    if app_name != None and not (app_name in m.group(0)):
240      return
241    if m.group(1) == 'mm_filemap_add_to_page_cache':
242      pagecache_stats.add_page(device_number, inode, m.group(4))
243    elif m.group(1) == 'mm_filemap_delete_from_page_cache':
244      pagecache_stats.remove_page(device_number, inode, m.group(4))
245
246def build_inode_lookup_table(inode_dump):
247  inode2filename = {}
248  text = inode_dump.splitlines()
249  for line in text:
250    result = re.match('([0-9]+)d? ([0-9]+) ([0-9]+) (.*)', line)
251    if result:
252      inode2filename[(int(result.group(1)), int(result.group(2)))] = (result.group(4), result.group(3))
253
254  return inode2filename;
255
256def get_inode_data(datafile, dumpfile, adb_serial):
257  if datafile is not None and os.path.isfile(datafile):
258    print('Using cached inode data from ' + datafile)
259    f = open(datafile, 'r')
260    stat_dump = f.read();
261  else:
262    # Build inode maps if we were tracing page cache
263    print('Downloading inode data from device')
264    stat_dump = AdbUtils.do_preprocess_adb_cmd(
265        'find /apex /system /system_ext /product /data /vendor ' +
266        '-exec stat -c "%d %i %s %n" {} \;', adb_serial)
267    if stat_dump is None:
268      print('Could not retrieve inode data from device.')
269      sys.exit(1)
270
271    if dumpfile is not None:
272      print('Storing inode data in ' + dumpfile)
273      f = open(dumpfile, 'w')
274      f.write(stat_dump)
275      f.close()
276
277    sys.stdout.write('Done.\n')
278
279  return stat_dump
280
281def read_and_parse_trace_file(trace_file, pagecache_stats, app_name):
282  for line in trace_file:
283    parse_atrace_line(line, pagecache_stats, app_name)
284  pagecache_stats.print_stats();
285
286def read_and_parse_trace_data_live(stdout, stderr, pagecache_stats, app_name):
287  # Start reading trace data
288  stdout_queue = queue.Queue(maxsize=128)
289  stderr_queue = queue.Queue()
290
291  stdout_thread = FileReaderThread(stdout, stdout_queue,
292                                   text_file=True, chunk_size=64)
293  stderr_thread = FileReaderThread(stderr, stderr_queue,
294                                   text_file=True)
295  stdout_thread.start()
296  stderr_thread.start()
297
298  stdscr = curses.initscr()
299
300  try:
301    height, width = stdscr.getmaxyx()
302    curses.noecho()
303    curses.cbreak()
304    stdscr.keypad(True)
305    stdscr.nodelay(True)
306    stdscr.refresh()
307    # We need at least a 30x100 window
308    used_width = max(width, 100)
309    used_height = max(height, 30)
310
311    # Create a pad for pagecache stats
312    pagecache_pad = curses.newpad(used_height - 2, used_width)
313
314    stdscr.addstr(used_height - 1, 0, 'KEY SHORTCUTS: (r)eset stats, CTRL-c to quit')
315    while (stdout_thread.isAlive() or stderr_thread.isAlive() or
316           not stdout_queue.empty() or not stderr_queue.empty()):
317      while not stderr_queue.empty():
318        # Pass along errors from adb.
319        line = stderr_queue.get().decode("utf-8")
320        sys.stderr.write(line)
321      while True:
322        try:
323          line = stdout_queue.get(True, STATS_UPDATE_INTERVAL).decode("utf-8")
324          parse_atrace_line(line, pagecache_stats, app_name)
325        except (queue.Empty, KeyboardInterrupt):
326          break
327
328      key = ''
329      try:
330        key = stdscr.getkey()
331      except:
332        pass
333
334      if key == 'r':
335        pagecache_stats.reset_stats()
336
337      pagecache_stats.print_stats_curses(pagecache_pad)
338  except Exception as e:
339    curses.endwin()
340    print(e)
341  finally:
342    curses.endwin()
343    # The threads should already have stopped, so this is just for cleanup.
344    stdout_thread.join()
345    stderr_thread.join()
346
347    stdout.close()
348    stderr.close()
349
350def parse_options(argv):
351  usage = 'Usage: %prog [options]'
352  desc = 'Example: %prog'
353  parser = optparse.OptionParser(usage=usage, description=desc)
354  parser.add_option('-d', dest='inode_dump_file', metavar='FILE',
355                    help='Dump the inode data read from a device to a file.'
356                    ' This file can then be reused with the -i option to speed'
357                    ' up future invocations of this script.')
358  parser.add_option('-i', dest='inode_data_file', metavar='FILE',
359                    help='Read cached inode data from a file saved arlier with the'
360                    ' -d option.')
361  parser.add_option('-s', '--serial', dest='device_serial', type='string',
362                    help='adb device serial number')
363  parser.add_option('-f', dest='trace_file', metavar='FILE',
364                    help='Show stats from a trace file, instead of running live.')
365  parser.add_option('-a', dest='app_name', type='string',
366                    help='filter a particular app')
367
368  options, categories = parser.parse_args(argv[1:])
369  if options.inode_dump_file and options.inode_data_file:
370    parser.error('options -d and -i can\'t be used at the same time')
371  return (options, categories)
372
373def main():
374  options, categories = parse_options(sys.argv)
375
376  # Load inode data for this device
377  inode_data = get_inode_data(options.inode_data_file, options.inode_dump_file,
378      options.device_serial)
379  # Build (dev, inode) -> filename hash
380  inode_lookup_table = build_inode_lookup_table(inode_data)
381  # Init pagecache stats
382  pagecache_stats = PagecacheStats(inode_lookup_table)
383
384  if options.trace_file is not None:
385    if not os.path.isfile(options.trace_file):
386      print('Couldn\'t load trace file.', file=sys.stderr)
387      sys.exit(1)
388    trace_file = open(options.trace_file, 'r')
389    read_and_parse_trace_file(trace_file, pagecache_stats, options.app_name)
390  else:
391    # Construct and execute trace command
392    trace_cmd = AdbUtils.construct_adb_shell_command(['atrace', '--stream', 'pagecache'],
393        options.device_serial)
394
395    try:
396      atrace = subprocess.Popen(trace_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
397          stderr=subprocess.PIPE)
398    except OSError as error:
399      print('The command failed', file=sys.stderr)
400      sys.exit(1)
401
402    read_and_parse_trace_data_live(atrace.stdout, atrace.stderr, pagecache_stats, options.app_name)
403
404if __name__ == "__main__":
405  main()
406