[Android] Revise recovery logic in device_status_check.

build/android/buildbot/bb_device_status_check.py

 #!/usr/bin/env python
 #
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A class to keep track of devices across builds and report state."""
 import json
 import logging
 import optparse
@@ skipping 206 matching lines @@
       except (psutil.NoSuchProcess, psutil.AccessDenied):
         pass
   for p in GetAllAdb():
     try:
       logging.error('Unable to kill %d (%s [%s])', p.pid, p.name,
                     ' '.join(p.cmdline))
     except (psutil.NoSuchProcess, psutil.AccessDenied):
       pass
 
 
-def main():
-  parser = optparse.OptionParser()
-  parser.add_option('', '--out-dir',
-                    help='Directory where the device path is stored',
-                    default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
-  parser.add_option('--no-provisioning-check', action='store_true',
-                    help='Will not check if devices are provisioned properly.')
-  parser.add_option('--device-status-dashboard', action='store_true',
-                    help='Output device status data for dashboard.')
-  parser.add_option('--restart-usb', action='store_true',
-                    help='DEPRECATED. '
-                         'This script now always tries to reset USB.')
-  parser.add_option('--json-output',
-                    help='Output JSON information into a specified file.')
-  parser.add_option('-v', '--verbose', action='count', default=1,
-                    help='Log more information.')
-
-  options, args = parser.parse_args()
-  if args:
-    parser.error('Unknown options %s' % args)
-
-  run_tests_helper.SetLogLevel(options.verbose)
-
+def RecoverDevices(args):
   # Remove the last build's "bad devices" before checking device statuses.
   device_blacklist.ResetBlacklist()
 
+  previous_devices = set(a.GetDeviceSerial()
+                         for a in adb_wrapper.AdbWrapper.Devices())
+
   KillAllAdb()
   reset_usb.reset_all_android_devices()
 
   try:
     expected_devices = set(device_list.GetPersistentDeviceList(
-        os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME)))
+        os.path.join(args.out_dir, device_list.LAST_DEVICES_FILENAME)))
   except IOError:
     expected_devices = set()
 
-  def all_devices_found():
-    devices = device_utils.DeviceUtils.HealthyDevices()
-    device_serials = set(d.adb.GetDeviceSerial() for d in devices)
-    return not bool(expected_devices.difference(device_serials))
+  all_devices = [device_utils.DeviceUtils(d)
+                 for d in previous_devices.union(expected_devices)]
 
-  timeout_retry.WaitFor(all_devices_found, wait_period=1, max_tries=5)
+  def blacklisting_recovery(device):
+    try:
+      device.WaitUntilFullyBooted()
+    except device_errors.CommandFailedError:
+      logging.exception('Failure while waiting for %s. Adding to blacklist.',
+                        str(device))
+      device_blacklist.ExtendBlacklist([str(device)])
+    except device_errors.CommandTimeoutError:
+      logging.exception('Timed out while waiting for %s. Adding to blacklist.',
+                        str(device))
+      device_blacklist.ExtendBlacklist([str(device)])
+
+  device_utils.DeviceUtils.parallel(all_devices).pMap(blacklisting_recovery)
 
   devices = device_utils.DeviceUtils.HealthyDevices()
   device_serials = set(d.adb.GetDeviceSerial() for d in devices)
 
   missing_devices = expected_devices.difference(device_serials)
   new_devices = device_serials.difference(expected_devices)
 
   if missing_devices or new_devices:
     logging.warning('expected_devices:')
     for d in sorted(expected_devices):
       logging.warning('  %s', d)
     logging.warning('devices:')
     for d in sorted(device_serials):
       logging.warning('  %s', d)
 
+  return devices
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option('', '--out-dir',
+                    help='Directory where the device path is stored',
+                    default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
+  parser.add_option('--no-provisioning-check', action='store_true',
+                    help='Will not check if devices are provisioned properly.')
+  parser.add_option('--device-status-dashboard', action='store_true',
+                    help='Output device status data for dashboard.')
+  parser.add_option('--restart-usb', action='store_true',
+                    help='DEPRECATED. '
+                         'This script now always tries to reset USB.')
+  parser.add_option('--json-output',
+                    help='Output JSON information into a specified file.')
+  parser.add_option('-v', '--verbose', action='count', default=1,
+                    help='Log more information.')
+
+  options, args = parser.parse_args()
+  if args:
+    parser.error('Unknown options %s' % args)
+
+  run_tests_helper.SetLogLevel(options.verbose)
+
+  devices = RecoverDevices(options)
+
   types, builds, batteries, errors, devices_ok, json_data = (
       [], [], [], [], [], [])
   if devices:
     types, builds, batteries, errors, devices_ok, json_data = (
         zip(*[DeviceInfo(dev, options) for dev in devices]))
 
   # Write device info to file for buildbot info display.
   if os.path.exists('/home/chrome-bot'):
     with open('/home/chrome-bot/.adb_device_info', 'w') as f:
       for device in json_data:
@@ skipping 67 matching lines @@
 
   if num_failed_devs == len(devices):
     return 2
 
   if not devices:
     return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())