Revert of [Android] Use devil for status and recovery in bb_device_status_check.

build/android/buildbot/bb_device_status_check.py

Index: build/android/buildbot/bb_device_status_check.py
diff --git a/build/android/buildbot/bb_device_status_check.py b/build/android/buildbot/bb_device_status_check.py
index 5ac9e8416d5deaa53a2fc68434a3b13fd8a18c9a..0e4107807956c3f2ff2403f63ca528d9ae1fb911 100755
--- a/build/android/buildbot/bb_device_status_check.py
+++ b/build/android/buildbot/bb_device_status_check.py
@@ -10,34 +10,314 @@
 import json
 import logging
 import os
+import psutil
 import re
+import signal
 import sys
 
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 import devil_chromium
+from devil.android import battery_utils
 from devil.android import device_blacklist
+from devil.android import device_errors
 from devil.android import device_list
 from devil.android import device_utils
-from devil.android.tools import device_recovery
-from devil.android.tools import device_status
+from devil.android.sdk import adb_wrapper
 from devil.constants import exit_codes
 from devil.utils import lsusb
+from devil.utils import reset_usb
 from devil.utils import run_tests_helper
 from pylib.constants import host_paths
 
 _RE_DEVICE_ID = re.compile(r'Device ID = (\d+)')
 
 
+def KillAllAdb():
+  def GetAllAdb():
+    for p in psutil.process_iter():
+      try:
+        if 'adb' in p.name:
+          yield p
+      except (psutil.NoSuchProcess, psutil.AccessDenied):
+        pass
+
+  for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
+    for p in GetAllAdb():
+      try:
+        logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
+                     ' '.join(p.cmdline))
+        p.send_signal(sig)
+      except (psutil.NoSuchProcess, psutil.AccessDenied):
+        pass
+  for p in GetAllAdb():
+    try:
+      logging.error('Unable to kill %d (%s [%s])', p.pid, p.name,
+                    ' '.join(p.cmdline))
+    except (psutil.NoSuchProcess, psutil.AccessDenied):
+      pass
+
+
+def _IsBlacklisted(serial, blacklist):
+  return blacklist and serial in blacklist.Read()
+
+
+def _BatteryStatus(device, blacklist):
+  battery_info = {}
+  try:
+    battery = battery_utils.BatteryUtils(device)
+    battery_info = battery.GetBatteryInfo(timeout=5)
+    battery_level = int(battery_info.get('level', 100))
+
+    if battery_level < 15:
+      logging.error('Critically low battery level (%d)', battery_level)
+      battery = battery_utils.BatteryUtils(device)
+      if not battery.GetCharging():
+        battery.SetCharging(True)
+      if blacklist:
+        blacklist.Extend([device.adb.GetDeviceSerial()], reason='low_battery')
+
+  except device_errors.CommandFailedError:
+    logging.exception('Failed to get battery information for %s',
+                      str(device))
+
+  return battery_info
+
+
+def _IMEISlice(device):
+  imei_slice = ''
+  try:
+    for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'],
+                                    check_return=True, timeout=5):
+      m = _RE_DEVICE_ID.match(l)
+      if m:
+        imei_slice = m.group(1)[-6:]
+  except device_errors.CommandFailedError:
+    logging.exception('Failed to get IMEI slice for %s', str(device))
+
+  return imei_slice
+
+
+def DeviceStatus(devices, blacklist):
+  """Generates status information for the given devices.
+
+  Args:
+    devices: The devices to generate status for.
+    blacklist: The current device blacklist.
+  Returns:
+    A dict of the following form:
+    {
+      '<serial>': {
+        'serial': '<serial>',
+        'adb_status': str,
+        'usb_status': bool,
+        'blacklisted': bool,
+        # only if the device is connected and not blacklisted
+        'type': ro.build.product,
+        'build': ro.build.id,
+        'build_detail': ro.build.fingerprint,
+        'battery': {
+          ...
+        },
+        'imei_slice': str,
+        'wifi_ip': str,
+      },
+      ...
+    }
+  """
+  adb_devices = {
+    a[0].GetDeviceSerial(): a
+    for a in adb_wrapper.AdbWrapper.Devices(desired_state=None, long_list=True)
+  }
+  usb_devices = set(lsusb.get_android_devices())
+
+  def blacklisting_device_status(device):
+    serial = device.adb.GetDeviceSerial()
+    adb_status = (
+        adb_devices[serial][1] if serial in adb_devices
+        else 'missing')
+    usb_status = bool(serial in usb_devices)
+
+    device_status = {
+      'serial': serial,
+      'adb_status': adb_status,
+      'usb_status': usb_status,
+    }
+
+    if not _IsBlacklisted(serial, blacklist):
+      if adb_status == 'device':
+        try:
+          build_product = device.build_product
+          build_id = device.build_id
+          build_fingerprint = device.GetProp('ro.build.fingerprint', cache=True)
+          wifi_ip = device.GetProp('dhcp.wlan0.ipaddress')
+          battery_info = _BatteryStatus(device, blacklist)
+          imei_slice = _IMEISlice(device)
+
+          if (device.product_name == 'mantaray' and
+              battery_info.get('AC powered', None) != 'true'):
+            logging.error('Mantaray device not connected to AC power.')
+
+          device_status.update({
+            'ro.build.product': build_product,
+            'ro.build.id': build_id,
+            'ro.build.fingerprint': build_fingerprint,
+            'battery': battery_info,
+            'imei_slice': imei_slice,
+            'wifi_ip': wifi_ip,
+
+            # TODO(jbudorick): Remove these once no clients depend on them.
+            'type': build_product,
+            'build': build_id,
+            'build_detail': build_fingerprint,
+          })
+
+        except device_errors.CommandFailedError:
+          logging.exception('Failure while getting device status for %s.',
+                            str(device))
+          if blacklist:
+            blacklist.Extend([serial], reason='status_check_failure')
+
+        except device_errors.CommandTimeoutError:
+          logging.exception('Timeout while getting device status for %s.',
+                            str(device))
+          if blacklist:
+            blacklist.Extend([serial], reason='status_check_timeout')
+
+      elif blacklist:
+        blacklist.Extend([serial],
+                         reason=adb_status if usb_status else 'offline')
+
+    device_status['blacklisted'] = _IsBlacklisted(serial, blacklist)
+
+    return device_status
+
+  parallel_devices = device_utils.DeviceUtils.parallel(devices)
+  statuses = parallel_devices.pMap(blacklisting_device_status).pGet(None)
+  return statuses
+
+
+def RecoverDevices(devices, blacklist):
+  """Attempts to recover any inoperable devices in the provided list.
+
+  Args:
+    devices: The list of devices to attempt to recover.
+    blacklist: The current device blacklist, which will be used then
+      reset.
+  Returns:
+    Nothing.
+  """
+
+  statuses = DeviceStatus(devices, blacklist)
+
+  should_restart_usb = set(
+      status['serial'] for status in statuses
+      if (not status['usb_status']
+          or status['adb_status'] in ('offline', 'missing')))
+  should_restart_adb = should_restart_usb.union(set(
+      status['serial'] for status in statuses
+      if status['adb_status'] == 'unauthorized'))
+  should_reboot_device = should_restart_adb.union(set(
+      status['serial'] for status in statuses
+      if status['blacklisted']))
+
+  logging.debug('Should restart USB for:')
+  for d in should_restart_usb:
+    logging.debug('  %s', d)
+  logging.debug('Should restart ADB for:')
+  for d in should_restart_adb:
+    logging.debug('  %s', d)
+  logging.debug('Should reboot:')
+  for d in should_reboot_device:
+    logging.debug('  %s', d)
+
+  if blacklist:
+    blacklist.Reset()
+
+  if should_restart_adb:
+    KillAllAdb()
+  for serial in should_restart_usb:
+    try:
+      reset_usb.reset_android_usb(serial)
+    except IOError:
+      logging.exception('Unable to reset USB for %s.', serial)
+      if blacklist:
+        blacklist.Extend([serial], reason='usb_failure')
+    except device_errors.DeviceUnreachableError:
+      logging.exception('Unable to reset USB for %s.', serial)
+      if blacklist:
+        blacklist.Extend([serial], reason='offline')
+
+  def blacklisting_recovery(device):
+    if _IsBlacklisted(device.adb.GetDeviceSerial(), blacklist):
+      logging.debug('%s is blacklisted, skipping recovery.', str(device))
+      return
+
+    if str(device) in should_reboot_device:
+      try:
+        device.WaitUntilFullyBooted(retries=0)
+        return
+      except (device_errors.CommandTimeoutError,
+              device_errors.CommandFailedError):
+        logging.exception('Failure while waiting for %s. '
+                          'Attempting to recover.', str(device))
+
+      try:
+        try:
+          device.Reboot(block=False, timeout=5, retries=0)
+        except device_errors.CommandTimeoutError:
+          logging.warning('Timed out while attempting to reboot %s normally.'
+                          'Attempting alternative reboot.', str(device))
+          # The device drops offline before we can grab the exit code, so
+          # we don't check for status.
+          device.adb.Root()
+          device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None,
+                           timeout=5, retries=0)
+      except device_errors.CommandFailedError:
+        logging.exception('Failed to reboot %s.', str(device))
+        if blacklist:
+          blacklist.Extend([device.adb.GetDeviceSerial()],
+                           reason='reboot_failure')
+      except device_errors.CommandTimeoutError:
+        logging.exception('Timed out while rebooting %s.', str(device))
+        if blacklist:
+          blacklist.Extend([device.adb.GetDeviceSerial()],
+                           reason='reboot_timeout')
+
+      try:
+        device.WaitUntilFullyBooted(retries=0)
+      except device_errors.CommandFailedError:
+        logging.exception('Failure while waiting for %s.', str(device))
+        if blacklist:
+          blacklist.Extend([device.adb.GetDeviceSerial()],
+                           reason='reboot_failure')
+      except device_errors.CommandTimeoutError:
+        logging.exception('Timed out while waiting for %s.', str(device))
+        if blacklist:
+          blacklist.Extend([device.adb.GetDeviceSerial()],
+                           reason='reboot_timeout')
+
+  device_utils.DeviceUtils.parallel(devices).pMap(blacklisting_recovery)
+
+
 def main():
   parser = argparse.ArgumentParser()
-  device_status.AddArguments(parser)
   parser.add_argument('--out-dir',
-                      help='DEPRECATED. '
-                           'Directory where the device path is stored',
+                      help='Directory where the device path is stored',
                       default=os.path.join(host_paths.DIR_SOURCE_ROOT, 'out'))
   parser.add_argument('--restart-usb', action='store_true',
                       help='DEPRECATED. '
                            'This script now always tries to reset USB.')
+  parser.add_argument('--json-output',
+                      help='Output JSON information into a specified file.')
+  parser.add_argument('--adb-path', type=os.path.abspath,
+                      help='Absolute path to the adb binary to use.')
+  parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
+  parser.add_argument('--known-devices-file', action='append', default=[],
+                      dest='known_devices_files',
+                      help='Path to known device lists.')
+  parser.add_argument('-v', '--verbose', action='count', default=1,
+                      help='Log more information.')
+
   args = parser.parse_args()
 
   run_tests_helper.SetLogLevel(args.verbose)
@@ -68,8 +348,8 @@
   devices = [device_utils.DeviceUtils(s)
              for s in expected_devices.union(usb_devices)]
 
-  device_recovery.RecoverDevices(devices, blacklist)
-  statuses = device_status.DeviceStatus(devices, blacklist)
+  RecoverDevices(devices, blacklist)
+  statuses = DeviceStatus(devices, blacklist)
 
   # Log the state of all devices.
   for status in statuses:
@@ -128,10 +408,9 @@
     with open(args.json_output, 'wb') as f:
       f.write(json.dumps(statuses, indent=4))
 
-  live_devices = [
-      status['serial'] for status in statuses
-      if (status['adb_status'] == 'device'
-          and not device_status.IsBlacklisted(status['serial'], blacklist))]
+  live_devices = [status['serial'] for status in statuses
+                  if (status['adb_status'] == 'device'
+                      and not _IsBlacklisted(status['serial'], blacklist))]
 
   # If all devices failed, or if there are no devices, it's an infra error.
   return 0 if live_devices else exit_codes.INFRA