xref: /aosp_15_r20/external/chromium-trace/catapult/devil/devil/android/tools/device_recovery.py (revision 1fa4b3da657c0e9ad43c0220bacf9731820715a5) 
1#!/usr/bin/env vpython
2# Copyright 2016 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""A script to recover devices in a known bad state."""
6
7import argparse
8import glob
9import logging
10import os
11import signal
12import sys
13
14import psutil
15
16if __name__ == '__main__':
17  sys.path.append(
18      os.path.abspath(
19          os.path.join(os.path.dirname(__file__), '..', '..', '..')))
20from devil.android import device_denylist
21from devil.android import device_errors
22from devil.android import device_utils
23from devil.android.sdk import adb_wrapper
24from devil.android.tools import device_status
25from devil.android.tools import script_common
26from devil.utils import logging_common
27from devil.utils import lsusb
28# TODO(jbudorick): Resolve this after experimenting w/ disabling the USB reset.
29from devil.utils import reset_usb  # pylint: disable=unused-import
30
31logger = logging.getLogger(__name__)
32
33from py_utils import modules_util
34
35# Script depends on features from psutil version 2.0 or higher.
36modules_util.RequireVersion(psutil, '2.0')
37
38
39def KillAllAdb():
40  def get_all_adb():
41    for p in psutil.process_iter():
42      try:
43        # Retrieve all required process infos at once.
44        pinfo = p.as_dict(attrs=['pid', 'name', 'cmdline'])
45        if pinfo['name'] == 'adb':
46          pinfo['cmdline'] = ' '.join(pinfo['cmdline'])
47          yield p, pinfo
48      except (psutil.NoSuchProcess, psutil.AccessDenied):
49        pass
50
51  for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
52    for p, pinfo in get_all_adb():
53      try:
54        pinfo['signal'] = sig
55        logger.info('kill %(signal)s %(pid)s (%(name)s [%(cmdline)s])', pinfo)
56        p.send_signal(sig)
57      except (psutil.NoSuchProcess, psutil.AccessDenied):
58        pass
59  for _, pinfo in get_all_adb():
60    try:
61      logger.error('Unable to kill %(pid)s (%(name)s [%(cmdline)s])', pinfo)
62    except (psutil.NoSuchProcess, psutil.AccessDenied):
63      pass
64
65
66def TryAuth(device):
67  """Uses anything in ~/.android/ that looks like a key to auth with the device.
68
69  Args:
70    device: The DeviceUtils device to attempt to auth.
71
72  Returns:
73    True if device successfully authed.
74  """
75  possible_keys = glob.glob(os.path.join(adb_wrapper.ADB_HOST_KEYS_DIR, '*key'))
76  if len(possible_keys) <= 1:
77    logger.warning('Only %d ADB keys available. Not forcing auth.',
78                   len(possible_keys))
79    return False
80
81  KillAllAdb()
82  adb_wrapper.AdbWrapper.StartServer(keys=possible_keys)
83  new_state = device.adb.GetState()
84  if new_state != 'device':
85    logger.error('Auth failed. Device %s still stuck in %s.', str(device),
86                 new_state)
87    return False
88
89  # It worked! Now register the host's default ADB key on the device so we don't
90  # have to do all that again.
91  pub_key = os.path.join(adb_wrapper.ADB_HOST_KEYS_DIR, 'adbkey.pub')
92  if not os.path.exists(pub_key):  # This really shouldn't happen.
93    logger.error('Default ADB key not available at %s.', pub_key)
94    return False
95
96  with open(pub_key) as f:
97    pub_key_contents = f.read()
98  try:
99    device.WriteFile(adb_wrapper.ADB_KEYS_FILE, pub_key_contents, as_root=True)
100  except (device_errors.CommandTimeoutError, device_errors.CommandFailedError,
101          device_errors.DeviceUnreachableError):
102    logger.exception('Unable to write default ADB key to %s.', str(device))
103    return False
104  return True
105
106
107def RecoverDevice(device, denylist, should_reboot=lambda device: True):
108  if device_status.IsDenylisted(device.adb.GetDeviceSerial(), denylist):
109    logger.debug('%s is denylisted, skipping recovery.', str(device))
110    return
111
112  if device.adb.GetState() == 'unauthorized' and TryAuth(device):
113    logger.info('Successfully authed device %s!', str(device))
114    return
115
116  if should_reboot(device):
117    should_restore_root = device.HasRoot()
118    try:
119      device.WaitUntilFullyBooted(retries=0)
120    except (device_errors.CommandTimeoutError, device_errors.CommandFailedError,
121            device_errors.DeviceUnreachableError):
122      logger.exception(
123          'Failure while waiting for %s. '
124          'Attempting to recover.', str(device))
125    try:
126      try:
127        device.Reboot(block=False, timeout=5, retries=0)
128      except device_errors.CommandTimeoutError:
129        logger.warning(
130            'Timed out while attempting to reboot %s normally.'
131            'Attempting alternative reboot.', str(device))
132        # The device drops offline before we can grab the exit code, so
133        # we don't check for status.
134        try:
135          device.adb.Root()
136        finally:
137          # We are already in a failure mode, attempt to reboot regardless of
138          # what device.adb.Root() returns. If the sysrq reboot fails an
139          # exception willbe thrown at that level.
140          device.adb.Shell(
141              'echo b > /proc/sysrq-trigger',
142              expect_status=None,
143              timeout=5,
144              retries=0)
145    except (device_errors.CommandFailedError,
146            device_errors.DeviceUnreachableError):
147      logger.exception('Failed to reboot %s.', str(device))
148      if denylist:
149        denylist.Extend([device.adb.GetDeviceSerial()], reason='reboot_failure')
150    except device_errors.CommandTimeoutError:
151      logger.exception('Timed out while rebooting %s.', str(device))
152      if denylist:
153        denylist.Extend([device.adb.GetDeviceSerial()], reason='reboot_timeout')
154
155    try:
156      device.WaitUntilFullyBooted(
157          retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT)
158      if should_restore_root:
159        device.EnableRoot()
160    except (device_errors.CommandFailedError,
161            device_errors.DeviceUnreachableError):
162      logger.exception('Failure while waiting for %s.', str(device))
163      if denylist:
164        denylist.Extend([device.adb.GetDeviceSerial()], reason='reboot_failure')
165    except device_errors.CommandTimeoutError:
166      logger.exception('Timed out while waiting for %s.', str(device))
167      if denylist:
168        denylist.Extend([device.adb.GetDeviceSerial()], reason='reboot_timeout')
169
170
171def RecoverDevices(devices, denylist, enable_usb_reset=False):
172  """Attempts to recover any inoperable devices in the provided list.
173
174  Args:
175    devices: The list of devices to attempt to recover.
176    denylist: The current device denylist, which will be used then
177      reset.
178  """
179
180  statuses = device_status.DeviceStatus(devices, denylist)
181
182  should_restart_usb = set(
183      status['serial'] for status in statuses
184      if (not status['usb_status'] or status['adb_status'] in ('offline',
185                                                               'missing')))
186  should_restart_adb = should_restart_usb.union(
187      set(status['serial'] for status in statuses
188          if status['adb_status'] == 'unauthorized'))
189  should_reboot_device = should_restart_usb.union(
190      set(status['serial'] for status in statuses if status['denylisted']))
191
192  logger.debug('Should restart USB for:')
193  for d in should_restart_usb:
194    logger.debug('  %s', d)
195  logger.debug('Should restart ADB for:')
196  for d in should_restart_adb:
197    logger.debug('  %s', d)
198  logger.debug('Should reboot:')
199  for d in should_reboot_device:
200    logger.debug('  %s', d)
201
202  if denylist:
203    denylist.Reset()
204
205  if should_restart_adb:
206    KillAllAdb()
207    adb_wrapper.AdbWrapper.StartServer()
208
209  for serial in should_restart_usb:
210    try:
211      # TODO(crbug.com/642194): Resetting may be causing more harm
212      # (specifically, kernel panics) than it does good.
213      if enable_usb_reset:
214        reset_usb.reset_android_usb(serial)
215      else:
216        logger.warning('USB reset disabled for %s (crbug.com/642914)', serial)
217    except IOError:
218      logger.exception('Unable to reset USB for %s.', serial)
219      if denylist:
220        denylist.Extend([serial], reason='USB failure')
221    except device_errors.DeviceUnreachableError:
222      logger.exception('Unable to reset USB for %s.', serial)
223      if denylist:
224        denylist.Extend([serial], reason='offline')
225
226  device_utils.DeviceUtils.parallel(devices).pMap(
227      RecoverDevice,
228      denylist,
229      should_reboot=lambda device: device.serial in should_reboot_device)
230
231
232def main():
233  parser = argparse.ArgumentParser()
234  logging_common.AddLoggingArguments(parser)
235  script_common.AddEnvironmentArguments(parser)
236  parser.add_argument('--denylist-file', help='Device denylist JSON file.')
237  parser.add_argument(
238      '--known-devices-file',
239      action='append',
240      default=[],
241      dest='known_devices_files',
242      help='Path to known device lists.')
243  parser.add_argument(
244      '--enable-usb-reset', action='store_true', help='Reset USB if necessary.')
245
246  args = parser.parse_args()
247  logging_common.InitializeLogging(args)
248  script_common.InitializeEnvironment(args)
249
250  denylist = (device_denylist.Denylist(args.denylist_file)
251              if args.denylist_file else None)
252
253  expected_devices = device_status.GetExpectedDevices(args.known_devices_files)
254  usb_devices = set(lsusb.get_android_devices())
255  devices = [
256      device_utils.DeviceUtils(s) for s in expected_devices.union(usb_devices)
257  ]
258
259  RecoverDevices(devices, denylist, enable_usb_reset=args.enable_usb_reset)
260
261
262if __name__ == '__main__':
263  sys.exit(main())
264