1*6777b538SAndroid Build Coastguard Worker#!/usr/bin/env vpython3 2*6777b538SAndroid Build Coastguard Worker# 3*6777b538SAndroid Build Coastguard Worker# Copyright 2013 The Chromium Authors 4*6777b538SAndroid Build Coastguard Worker# Use of this source code is governed by a BSD-style license that can be 5*6777b538SAndroid Build Coastguard Worker# found in the LICENSE file. 6*6777b538SAndroid Build Coastguard Worker# 7*6777b538SAndroid Build Coastguard Worker# Find the most recent tombstone file(s) on all connected devices 8*6777b538SAndroid Build Coastguard Worker# and prints their stacks. 9*6777b538SAndroid Build Coastguard Worker# 10*6777b538SAndroid Build Coastguard Worker# Assumes tombstone file was created with current symbols. 11*6777b538SAndroid Build Coastguard Worker 12*6777b538SAndroid Build Coastguard Workerimport argparse 13*6777b538SAndroid Build Coastguard Workerimport datetime 14*6777b538SAndroid Build Coastguard Workerimport logging 15*6777b538SAndroid Build Coastguard Workerimport os 16*6777b538SAndroid Build Coastguard Workerimport sys 17*6777b538SAndroid Build Coastguard Worker 18*6777b538SAndroid Build Coastguard Workerfrom multiprocessing.pool import ThreadPool 19*6777b538SAndroid Build Coastguard Worker 20*6777b538SAndroid Build Coastguard Workerimport devil_chromium 21*6777b538SAndroid Build Coastguard Worker 22*6777b538SAndroid Build Coastguard Workerfrom devil.android import device_denylist 23*6777b538SAndroid Build Coastguard Workerfrom devil.android import device_errors 24*6777b538SAndroid Build Coastguard Workerfrom devil.android import device_utils 25*6777b538SAndroid Build Coastguard Workerfrom devil.utils import run_tests_helper 26*6777b538SAndroid Build Coastguard Workerfrom pylib import constants 27*6777b538SAndroid Build Coastguard Workerfrom pylib.symbols import stack_symbolizer 28*6777b538SAndroid Build Coastguard Worker 29*6777b538SAndroid Build Coastguard Worker 30*6777b538SAndroid Build Coastguard Worker_TZ_UTC = {'TZ': 'UTC'} 31*6777b538SAndroid Build Coastguard Worker 32*6777b538SAndroid Build Coastguard Worker 33*6777b538SAndroid Build Coastguard Workerdef _ListTombstones(device): 34*6777b538SAndroid Build Coastguard Worker """List the tombstone files on the device. 35*6777b538SAndroid Build Coastguard Worker 36*6777b538SAndroid Build Coastguard Worker Args: 37*6777b538SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 38*6777b538SAndroid Build Coastguard Worker 39*6777b538SAndroid Build Coastguard Worker Yields: 40*6777b538SAndroid Build Coastguard Worker Tuples of (tombstone filename, date time of file on device). 41*6777b538SAndroid Build Coastguard Worker """ 42*6777b538SAndroid Build Coastguard Worker try: 43*6777b538SAndroid Build Coastguard Worker if not device.PathExists('/data/tombstones', as_root=True): 44*6777b538SAndroid Build Coastguard Worker return 45*6777b538SAndroid Build Coastguard Worker entries = device.StatDirectory('/data/tombstones', as_root=True) 46*6777b538SAndroid Build Coastguard Worker for entry in entries: 47*6777b538SAndroid Build Coastguard Worker if 'tombstone' in entry['filename']: 48*6777b538SAndroid Build Coastguard Worker yield (entry['filename'], 49*6777b538SAndroid Build Coastguard Worker datetime.datetime.fromtimestamp(entry['st_mtime'])) 50*6777b538SAndroid Build Coastguard Worker except device_errors.CommandFailedError: 51*6777b538SAndroid Build Coastguard Worker logging.exception('Could not retrieve tombstones.') 52*6777b538SAndroid Build Coastguard Worker except device_errors.DeviceUnreachableError: 53*6777b538SAndroid Build Coastguard Worker logging.exception('Device unreachable retrieving tombstones.') 54*6777b538SAndroid Build Coastguard Worker except device_errors.CommandTimeoutError: 55*6777b538SAndroid Build Coastguard Worker logging.exception('Timed out retrieving tombstones.') 56*6777b538SAndroid Build Coastguard Worker 57*6777b538SAndroid Build Coastguard Worker 58*6777b538SAndroid Build Coastguard Workerdef _GetDeviceDateTime(device): 59*6777b538SAndroid Build Coastguard Worker """Determine the date time on the device. 60*6777b538SAndroid Build Coastguard Worker 61*6777b538SAndroid Build Coastguard Worker Args: 62*6777b538SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 63*6777b538SAndroid Build Coastguard Worker 64*6777b538SAndroid Build Coastguard Worker Returns: 65*6777b538SAndroid Build Coastguard Worker A datetime instance. 66*6777b538SAndroid Build Coastguard Worker """ 67*6777b538SAndroid Build Coastguard Worker device_now_string = device.RunShellCommand( 68*6777b538SAndroid Build Coastguard Worker ['date'], check_return=True, env=_TZ_UTC) 69*6777b538SAndroid Build Coastguard Worker return datetime.datetime.strptime( 70*6777b538SAndroid Build Coastguard Worker device_now_string[0], '%a %b %d %H:%M:%S %Z %Y') 71*6777b538SAndroid Build Coastguard Worker 72*6777b538SAndroid Build Coastguard Worker 73*6777b538SAndroid Build Coastguard Workerdef _GetTombstoneData(device, tombstone_file): 74*6777b538SAndroid Build Coastguard Worker """Retrieve the tombstone data from the device 75*6777b538SAndroid Build Coastguard Worker 76*6777b538SAndroid Build Coastguard Worker Args: 77*6777b538SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 78*6777b538SAndroid Build Coastguard Worker tombstone_file: the tombstone to retrieve 79*6777b538SAndroid Build Coastguard Worker 80*6777b538SAndroid Build Coastguard Worker Returns: 81*6777b538SAndroid Build Coastguard Worker A list of lines 82*6777b538SAndroid Build Coastguard Worker """ 83*6777b538SAndroid Build Coastguard Worker return device.ReadFile( 84*6777b538SAndroid Build Coastguard Worker '/data/tombstones/' + tombstone_file, as_root=True).splitlines() 85*6777b538SAndroid Build Coastguard Worker 86*6777b538SAndroid Build Coastguard Worker 87*6777b538SAndroid Build Coastguard Workerdef _EraseTombstone(device, tombstone_file): 88*6777b538SAndroid Build Coastguard Worker """Deletes a tombstone from the device. 89*6777b538SAndroid Build Coastguard Worker 90*6777b538SAndroid Build Coastguard Worker Args: 91*6777b538SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 92*6777b538SAndroid Build Coastguard Worker tombstone_file: the tombstone to delete. 93*6777b538SAndroid Build Coastguard Worker """ 94*6777b538SAndroid Build Coastguard Worker return device.RunShellCommand( 95*6777b538SAndroid Build Coastguard Worker ['rm', '/data/tombstones/' + tombstone_file], 96*6777b538SAndroid Build Coastguard Worker as_root=True, check_return=True) 97*6777b538SAndroid Build Coastguard Worker 98*6777b538SAndroid Build Coastguard Worker 99*6777b538SAndroid Build Coastguard Workerdef _ResolveTombstone(args): 100*6777b538SAndroid Build Coastguard Worker tombstone = args[0] 101*6777b538SAndroid Build Coastguard Worker tombstone_symbolizer = args[1] 102*6777b538SAndroid Build Coastguard Worker lines = [] 103*6777b538SAndroid Build Coastguard Worker lines += [tombstone['file'] + ' created on ' + str(tombstone['time']) + 104*6777b538SAndroid Build Coastguard Worker ', about this long ago: ' + 105*6777b538SAndroid Build Coastguard Worker (str(tombstone['device_now'] - tombstone['time']) + 106*6777b538SAndroid Build Coastguard Worker ' Device: ' + tombstone['serial'])] 107*6777b538SAndroid Build Coastguard Worker logging.info('\n'.join(lines)) 108*6777b538SAndroid Build Coastguard Worker logging.info('Resolving...') 109*6777b538SAndroid Build Coastguard Worker lines += tombstone_symbolizer.ExtractAndResolveNativeStackTraces( 110*6777b538SAndroid Build Coastguard Worker tombstone['data'], 111*6777b538SAndroid Build Coastguard Worker tombstone['device_abi'], 112*6777b538SAndroid Build Coastguard Worker tombstone['stack']) 113*6777b538SAndroid Build Coastguard Worker return lines 114*6777b538SAndroid Build Coastguard Worker 115*6777b538SAndroid Build Coastguard Worker 116*6777b538SAndroid Build Coastguard Workerdef _ResolveTombstones(jobs, tombstones, tombstone_symbolizer): 117*6777b538SAndroid Build Coastguard Worker """Resolve a list of tombstones. 118*6777b538SAndroid Build Coastguard Worker 119*6777b538SAndroid Build Coastguard Worker Args: 120*6777b538SAndroid Build Coastguard Worker jobs: the number of jobs to use with multithread. 121*6777b538SAndroid Build Coastguard Worker tombstones: a list of tombstones. 122*6777b538SAndroid Build Coastguard Worker """ 123*6777b538SAndroid Build Coastguard Worker if not tombstones: 124*6777b538SAndroid Build Coastguard Worker logging.warning('No tombstones to resolve.') 125*6777b538SAndroid Build Coastguard Worker return [] 126*6777b538SAndroid Build Coastguard Worker if len(tombstones) == 1: 127*6777b538SAndroid Build Coastguard Worker data = [_ResolveTombstone([tombstones[0], tombstone_symbolizer])] 128*6777b538SAndroid Build Coastguard Worker else: 129*6777b538SAndroid Build Coastguard Worker pool = ThreadPool(jobs) 130*6777b538SAndroid Build Coastguard Worker data = pool.map( 131*6777b538SAndroid Build Coastguard Worker _ResolveTombstone, 132*6777b538SAndroid Build Coastguard Worker [[tombstone, tombstone_symbolizer] for tombstone in tombstones]) 133*6777b538SAndroid Build Coastguard Worker pool.close() 134*6777b538SAndroid Build Coastguard Worker pool.join() 135*6777b538SAndroid Build Coastguard Worker resolved_tombstones = [] 136*6777b538SAndroid Build Coastguard Worker for tombstone in data: 137*6777b538SAndroid Build Coastguard Worker resolved_tombstones.extend(tombstone) 138*6777b538SAndroid Build Coastguard Worker return resolved_tombstones 139*6777b538SAndroid Build Coastguard Worker 140*6777b538SAndroid Build Coastguard Worker 141*6777b538SAndroid Build Coastguard Workerdef _GetTombstonesForDevice(device, resolve_all_tombstones, 142*6777b538SAndroid Build Coastguard Worker include_stack_symbols, 143*6777b538SAndroid Build Coastguard Worker wipe_tombstones): 144*6777b538SAndroid Build Coastguard Worker """Returns a list of tombstones on a given device. 145*6777b538SAndroid Build Coastguard Worker 146*6777b538SAndroid Build Coastguard Worker Args: 147*6777b538SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 148*6777b538SAndroid Build Coastguard Worker resolve_all_tombstone: Whether to resolve every tombstone. 149*6777b538SAndroid Build Coastguard Worker include_stack_symbols: Whether to include symbols for stack data. 150*6777b538SAndroid Build Coastguard Worker wipe_tombstones: Whether to wipe tombstones. 151*6777b538SAndroid Build Coastguard Worker """ 152*6777b538SAndroid Build Coastguard Worker ret = [] 153*6777b538SAndroid Build Coastguard Worker all_tombstones = list(_ListTombstones(device)) 154*6777b538SAndroid Build Coastguard Worker if not all_tombstones: 155*6777b538SAndroid Build Coastguard Worker logging.warning('No tombstones.') 156*6777b538SAndroid Build Coastguard Worker return ret 157*6777b538SAndroid Build Coastguard Worker 158*6777b538SAndroid Build Coastguard Worker # Sort the tombstones in date order, descending 159*6777b538SAndroid Build Coastguard Worker all_tombstones.sort(key=lambda a: a[1], reverse=True) 160*6777b538SAndroid Build Coastguard Worker 161*6777b538SAndroid Build Coastguard Worker # Only resolve the most recent unless --all-tombstones given. 162*6777b538SAndroid Build Coastguard Worker tombstones = all_tombstones if resolve_all_tombstones else [all_tombstones[0]] 163*6777b538SAndroid Build Coastguard Worker 164*6777b538SAndroid Build Coastguard Worker device_now = _GetDeviceDateTime(device) 165*6777b538SAndroid Build Coastguard Worker try: 166*6777b538SAndroid Build Coastguard Worker for tombstone_file, tombstone_time in tombstones: 167*6777b538SAndroid Build Coastguard Worker ret += [{'serial': str(device), 168*6777b538SAndroid Build Coastguard Worker 'device_abi': device.product_cpu_abi, 169*6777b538SAndroid Build Coastguard Worker 'device_now': device_now, 170*6777b538SAndroid Build Coastguard Worker 'time': tombstone_time, 171*6777b538SAndroid Build Coastguard Worker 'file': tombstone_file, 172*6777b538SAndroid Build Coastguard Worker 'stack': include_stack_symbols, 173*6777b538SAndroid Build Coastguard Worker 'data': _GetTombstoneData(device, tombstone_file)}] 174*6777b538SAndroid Build Coastguard Worker except device_errors.CommandFailedError: 175*6777b538SAndroid Build Coastguard Worker for entry in device.StatDirectory( 176*6777b538SAndroid Build Coastguard Worker '/data/tombstones', as_root=True, timeout=60): 177*6777b538SAndroid Build Coastguard Worker logging.info('%s: %s', str(device), entry) 178*6777b538SAndroid Build Coastguard Worker raise 179*6777b538SAndroid Build Coastguard Worker 180*6777b538SAndroid Build Coastguard Worker # Erase all the tombstones if desired. 181*6777b538SAndroid Build Coastguard Worker if wipe_tombstones: 182*6777b538SAndroid Build Coastguard Worker for tombstone_file, _ in all_tombstones: 183*6777b538SAndroid Build Coastguard Worker _EraseTombstone(device, tombstone_file) 184*6777b538SAndroid Build Coastguard Worker 185*6777b538SAndroid Build Coastguard Worker return ret 186*6777b538SAndroid Build Coastguard Worker 187*6777b538SAndroid Build Coastguard Worker 188*6777b538SAndroid Build Coastguard Workerdef ClearAllTombstones(device): 189*6777b538SAndroid Build Coastguard Worker """Clear all tombstones in the device. 190*6777b538SAndroid Build Coastguard Worker 191*6777b538SAndroid Build Coastguard Worker Args: 192*6777b538SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 193*6777b538SAndroid Build Coastguard Worker """ 194*6777b538SAndroid Build Coastguard Worker all_tombstones = list(_ListTombstones(device)) 195*6777b538SAndroid Build Coastguard Worker if not all_tombstones: 196*6777b538SAndroid Build Coastguard Worker logging.warning('No tombstones to clear.') 197*6777b538SAndroid Build Coastguard Worker 198*6777b538SAndroid Build Coastguard Worker for tombstone_file, _ in all_tombstones: 199*6777b538SAndroid Build Coastguard Worker _EraseTombstone(device, tombstone_file) 200*6777b538SAndroid Build Coastguard Worker 201*6777b538SAndroid Build Coastguard Worker 202*6777b538SAndroid Build Coastguard Workerdef ResolveTombstones(device, resolve_all_tombstones, include_stack_symbols, 203*6777b538SAndroid Build Coastguard Worker wipe_tombstones, jobs=4, apk_under_test=None, 204*6777b538SAndroid Build Coastguard Worker tombstone_symbolizer=None): 205*6777b538SAndroid Build Coastguard Worker """Resolve tombstones in the device. 206*6777b538SAndroid Build Coastguard Worker 207*6777b538SAndroid Build Coastguard Worker Args: 208*6777b538SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 209*6777b538SAndroid Build Coastguard Worker resolve_all_tombstone: Whether to resolve every tombstone. 210*6777b538SAndroid Build Coastguard Worker include_stack_symbols: Whether to include symbols for stack data. 211*6777b538SAndroid Build Coastguard Worker wipe_tombstones: Whether to wipe tombstones. 212*6777b538SAndroid Build Coastguard Worker jobs: Number of jobs to use when processing multiple crash stacks. 213*6777b538SAndroid Build Coastguard Worker 214*6777b538SAndroid Build Coastguard Worker Returns: 215*6777b538SAndroid Build Coastguard Worker A list of resolved tombstones. 216*6777b538SAndroid Build Coastguard Worker """ 217*6777b538SAndroid Build Coastguard Worker return _ResolveTombstones(jobs, 218*6777b538SAndroid Build Coastguard Worker _GetTombstonesForDevice(device, 219*6777b538SAndroid Build Coastguard Worker resolve_all_tombstones, 220*6777b538SAndroid Build Coastguard Worker include_stack_symbols, 221*6777b538SAndroid Build Coastguard Worker wipe_tombstones), 222*6777b538SAndroid Build Coastguard Worker (tombstone_symbolizer 223*6777b538SAndroid Build Coastguard Worker or stack_symbolizer.Symbolizer(apk_under_test))) 224*6777b538SAndroid Build Coastguard Worker 225*6777b538SAndroid Build Coastguard Worker 226*6777b538SAndroid Build Coastguard Workerdef main(): 227*6777b538SAndroid Build Coastguard Worker custom_handler = logging.StreamHandler(sys.stdout) 228*6777b538SAndroid Build Coastguard Worker custom_handler.setFormatter(run_tests_helper.CustomFormatter()) 229*6777b538SAndroid Build Coastguard Worker logging.getLogger().addHandler(custom_handler) 230*6777b538SAndroid Build Coastguard Worker logging.getLogger().setLevel(logging.INFO) 231*6777b538SAndroid Build Coastguard Worker 232*6777b538SAndroid Build Coastguard Worker parser = argparse.ArgumentParser() 233*6777b538SAndroid Build Coastguard Worker parser.add_argument('--device', 234*6777b538SAndroid Build Coastguard Worker help='The serial number of the device. If not specified ' 235*6777b538SAndroid Build Coastguard Worker 'will use all devices.') 236*6777b538SAndroid Build Coastguard Worker parser.add_argument('--denylist-file', help='Device denylist JSON file.') 237*6777b538SAndroid Build Coastguard Worker parser.add_argument('-a', '--all-tombstones', action='store_true', 238*6777b538SAndroid Build Coastguard Worker help='Resolve symbols for all tombstones, rather than ' 239*6777b538SAndroid Build Coastguard Worker 'just the most recent.') 240*6777b538SAndroid Build Coastguard Worker parser.add_argument('-s', '--stack', action='store_true', 241*6777b538SAndroid Build Coastguard Worker help='Also include symbols for stack data') 242*6777b538SAndroid Build Coastguard Worker parser.add_argument('-w', '--wipe-tombstones', action='store_true', 243*6777b538SAndroid Build Coastguard Worker help='Erase all tombstones from device after processing') 244*6777b538SAndroid Build Coastguard Worker parser.add_argument('-j', '--jobs', type=int, 245*6777b538SAndroid Build Coastguard Worker default=4, 246*6777b538SAndroid Build Coastguard Worker help='Number of jobs to use when processing multiple ' 247*6777b538SAndroid Build Coastguard Worker 'crash stacks.') 248*6777b538SAndroid Build Coastguard Worker parser.add_argument('--output-directory', 249*6777b538SAndroid Build Coastguard Worker help='Path to the root build directory.') 250*6777b538SAndroid Build Coastguard Worker parser.add_argument('--adb-path', type=os.path.abspath, 251*6777b538SAndroid Build Coastguard Worker help='Path to the adb binary.') 252*6777b538SAndroid Build Coastguard Worker args = parser.parse_args() 253*6777b538SAndroid Build Coastguard Worker 254*6777b538SAndroid Build Coastguard Worker if args.output_directory: 255*6777b538SAndroid Build Coastguard Worker constants.SetOutputDirectory(args.output_directory) 256*6777b538SAndroid Build Coastguard Worker 257*6777b538SAndroid Build Coastguard Worker devil_chromium.Initialize(output_directory=constants.GetOutDirectory(), 258*6777b538SAndroid Build Coastguard Worker adb_path=args.adb_path) 259*6777b538SAndroid Build Coastguard Worker 260*6777b538SAndroid Build Coastguard Worker denylist = (device_denylist.Denylist(args.denylist_file) 261*6777b538SAndroid Build Coastguard Worker if args.denylist_file else None) 262*6777b538SAndroid Build Coastguard Worker 263*6777b538SAndroid Build Coastguard Worker if args.device: 264*6777b538SAndroid Build Coastguard Worker devices = [device_utils.DeviceUtils(args.device)] 265*6777b538SAndroid Build Coastguard Worker else: 266*6777b538SAndroid Build Coastguard Worker devices = device_utils.DeviceUtils.HealthyDevices(denylist) 267*6777b538SAndroid Build Coastguard Worker 268*6777b538SAndroid Build Coastguard Worker # This must be done serially because strptime can hit a race condition if 269*6777b538SAndroid Build Coastguard Worker # used for the first time in a multithreaded environment. 270*6777b538SAndroid Build Coastguard Worker # http://bugs.python.org/issue7980 271*6777b538SAndroid Build Coastguard Worker for device in devices: 272*6777b538SAndroid Build Coastguard Worker resolved_tombstones = ResolveTombstones( 273*6777b538SAndroid Build Coastguard Worker device, args.all_tombstones, 274*6777b538SAndroid Build Coastguard Worker args.stack, args.wipe_tombstones, args.jobs) 275*6777b538SAndroid Build Coastguard Worker for line in resolved_tombstones: 276*6777b538SAndroid Build Coastguard Worker logging.info(line) 277*6777b538SAndroid Build Coastguard Worker 278*6777b538SAndroid Build Coastguard Worker 279*6777b538SAndroid Build Coastguard Workerif __name__ == '__main__': 280*6777b538SAndroid Build Coastguard Worker sys.exit(main()) 281