If device does not come back after provisioning, add to bad devices.

build/android/provision_devices.py

Index: build/android/provision_devices.py
diff --git a/build/android/provision_devices.py b/build/android/provision_devices.py
index 2c56b5817a65055abadb902fc16b1f5c9413ab4b..7a728ebb9f4589b1c7ef635bae97810fc7d7da09 100755
--- a/build/android/provision_devices.py
+++ b/build/android/provision_devices.py
@@ -21,6 +21,7 @@ import time
 from pylib import android_commands
 from pylib import constants
 from pylib import device_settings
+from pylib.device import device_blacklist
 from pylib.device import device_errors
 from pylib.device import device_utils
 
@@ -154,6 +155,46 @@ def WipeDevicesIfPossible(devices):
       device.WaitUntilFullyBooted(timeout=90)
 
 
+def ProvisionDevice(device_serial, is_perf, disable_location):
+  device = device_utils.DeviceUtils(device_serial)
+  device.old_interface.EnableAdbRoot()
+  _ConfigureLocalProperties(device, is_perf)
+  device_settings_map = device_settings.DETERMINISTIC_DEVICE_SETTINGS
+  if disable_location:
+    device_settings_map.update(device_settings.DISABLE_LOCATION_SETTING)
+  else:
+    device_settings_map.update(device_settings.ENABLE_LOCATION_SETTING)
+  device_settings.ConfigureContentSettingsDict(device, device_settings_map)
+  device_settings.SetLockScreenSettings(device)
+  if is_perf:
+    # TODO(tonyg): We eventually want network on. However, currently radios
+    # can cause perfbots to drain faster than they charge.
+    device_settings.ConfigureContentSettingsDict(
+      device, device_settings.NETWORK_DISABLED_SETTINGS)
+    # Some perf bots run benchmarks with USB charging disabled which leads
+    # to gradual draining of the battery. We must wait for a full charge
+    # before starting a run in order to keep the devices online.
+    try:
+      battery_info = device.old_interface.GetBatteryInfo()
+    except Exception as e:
+      battery_info = {}
+      logging.error('Unable to obtain battery info for %s, %s',
+                    device_serial, e)
+
+    while int(battery_info.get('level', 100)) < 95:
+      if not device.old_interface.IsDeviceCharging():
+        if device.old_interface.CanControlUsbCharging():
+          device.old_interface.EnableUsbCharging()
+        else:
+          logging.error('Device is not charging')
+          break
+      logging.info('Waiting for device to charge. Current level=%s',
+                     battery_info.get('level', 0))
+      time.sleep(60)
+      battery_info = device.old_interface.GetBatteryInfo()
+  device.RunShellCommand('date -u %f' % time.time(), as_root=True)
+
+
 def ProvisionDevices(options):
   is_perf = 'perf' in os.environ.get('BUILDBOT_BUILDERNAME', '').lower()
   # TODO(jbudorick): Parallelize provisioning of all attached devices after
@@ -167,55 +208,43 @@ def ProvisionDevices(options):
   if not options.skip_wipe:
     WipeDevicesIfPossible(devices)

[cjhopman, 2014/08/06 21:50:05]

I guess there is stuff in here, too, that could throw an exception and fail this
step for a single bad device

[navabi, 2014/08/06 21:58:51]

True. The original patch was for a particular place where this was failing (and
a P0 that was changed to a P1). We have not seen this be a problem in practice
here or on the reboot.

It doesn't hurt to add it here though.

[cjhopman, 2014/08/06 22:02:10]

Yeah, I think the main point is that one bad device shouldn't cause the whole
build to fail. But if we don't actually see the wipe cause such failures, then
I'm not too worried about it.

 
+  bad_devices = []
   # Provision devices
   for device_serial in devices:
-    device = device_utils.DeviceUtils(device_serial)
-    device.old_interface.EnableAdbRoot()
-    _ConfigureLocalProperties(device, is_perf)
-    device_settings_map = device_settings.DETERMINISTIC_DEVICE_SETTINGS
-    if options.disable_location:
-      device_settings_map.update(device_settings.DISABLE_LOCATION_SETTING)
-    else:
-      device_settings_map.update(device_settings.ENABLE_LOCATION_SETTING)
-    device_settings.ConfigureContentSettingsDict(device, device_settings_map)
-    device_settings.SetLockScreenSettings(device)
-    if is_perf:
-      # TODO(tonyg): We eventually want network on. However, currently radios
-      # can cause perfbots to drain faster than they charge.
-      device_settings.ConfigureContentSettingsDict(
-          device, device_settings.NETWORK_DISABLED_SETTINGS)
-      # Some perf bots run benchmarks with USB charging disabled which leads
-      # to gradual draining of the battery. We must wait for a full charge
-      # before starting a run in order to keep the devices online.
-      try:
-        battery_info = device.old_interface.GetBatteryInfo()
-      except Exception as e:
-        battery_info = {}
-        logging.error('Unable to obtain battery info for %s, %s',
-                      device_serial, e)
-
-      while int(battery_info.get('level', 100)) < 95:
-        if not device.old_interface.IsDeviceCharging():
-          if device.old_interface.CanControlUsbCharging():
-            device.old_interface.EnableUsbCharging()
-          else:
-            logging.error('Device is not charging')
-            break
-        logging.info('Waiting for device to charge. Current level=%s',
-                     battery_info.get('level', 0))
-        time.sleep(60)
-        battery_info = device.old_interface.GetBatteryInfo()
-    device.RunShellCommand('date -u %f' % time.time(), as_root=True)
+    try:
+      ProvisionDevice(device_serial, is_perf, options.disable_location)
+    except errors.WaitForResponseTimedOutError:
+      logging.info('Timed out waiting for device %s. Adding to blacklist.',
+                   device_serial)
+      bad_devices.append(device_serial)
+      # Device black list is reset by bb_device_status_check.py per build.
+      device_blacklist.ExtendBlacklist([device_serial])
+  devices = [device for device in devices if device not in bad_devices]
+
+  # If there are no good devices
+  if not devices:
+    raise device_errors.NoDevicesError
+
   try:
     device_utils.DeviceUtils.parallel(devices).Reboot(True)

[cjhopman, 2014/08/06 21:50:05]

This is just parallelizing sending the 'adb reboot' command? Is that really that
much benefit? This would be much simpler if there was just one loop over the
devices that did wipe-provision-reboot-wait (ideally we'd parrallelize that
whole sequence).

[navabi, 2014/08/06 21:58:51]

No. The True that is being passed to Reboot is to block for the reboot to
complete. See pylib/device/device_utils.py
Yes. Especially on the try bots where there are 8 devices and cycle time matters
the most.

[cjhopman, 2014/08/06 22:02:10]

Acknowledged.

   except errors.DeviceUnresponsiveError:
     pass
+
+  bad_devices = []
   for device_serial in devices:
     device = device_utils.DeviceUtils(device_serial)
-    device.WaitUntilFullyBooted(timeout=90)
-    (_, props) = device.old_interface.GetShellCommandStatusAndOutput('getprop')
-    for prop in props:
-      print prop
+    try:
+      device.WaitUntilFullyBooted(timeout=90)
+      (_, prop) = device.old_interface.GetShellCommandStatusAndOutput('getprop')
+      for p in prop:
+        print p
+    except errors.WaitForResponseTimedOutError:
+      logging.info('Timed out waiting for device %s. Adding to blacklist.',
+                   device_serial)
+      bad_devices.append(device_serial)
+      # Device black list is reset by bb_device_status_check.py per build.
+      device_blacklist.ExtendBlacklist([device_serial])
+  devices = [device for device in devices if device not in bad_devices]

[cjhopman, 2014/08/06 21:50:05]

This should now raise NoDevicesError if devices is empty?

[navabi, 2014/08/06 21:58:52]

Good catch. Done.

   if options.auto_reconnect:
     PushAndLaunchAdbReboot(devices, options.target)