Try to wait for devices to come back after a perf test failure.

build/android/pylib/perf/test_runner.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs perf tests.
 
 Our buildbot infrastructure requires each slave to run steps serially.
 This is sub-optimal for android, where these steps can run independently on
 multiple connected devices.
 
@@ skipping 43 matching lines @@
 import pickle
 import sys
 import threading
 import time
 
 from pylib import cmd_helper
 from pylib import constants
 from pylib import forwarder
 from pylib.base import base_test_result
 from pylib.base import base_test_runner
+from pylib.device import device_errors
 
 
 def OutputJsonList(json_input, json_output):
   with file(json_input, 'r') as i:
     all_steps = json.load(i)
   step_names = all_steps['steps'].keys()
   with file(json_output, 'w') as o:
     o.write(json.dumps(step_names))
   return 0
 
@@ skipping 171 matching lines @@
       output = str(e)
     finally:
       if self._options.single_step:
         logfile.stop()
     end_time = datetime.datetime.now()
     if exit_code is None:
       exit_code = -1
     logging.info('%s : exit_code=%d in %d secs at %s',
                  test_name, exit_code, (end_time - start_time).seconds,
                  self.device_serial)
-    result_type = base_test_result.ResultType.FAIL
+
     if exit_code == 0:
       result_type = base_test_result.ResultType.PASS
+    else:
+      result_type = base_test_result.ResultType.FAIL
+      # Since perf tests use device affinity, give the device a chance to
+      # recover if it is offline after a failure. Otherwise, the master sharder
+      # will remove it from the pool and future tests on this device will fail.
+      try:
+        self.device.WaitUntilFullyBooted(timeout=120)
+      except device_errors.CommandTimeoutError as e:
+        logging.error('Device failed to return after %s: %s' % (test_name, e))
+
     actual_exit_code = exit_code
     if test_name in self._flaky_tests:
       # The exit_code is used at the second stage when printing the
       # test output. If the test is flaky, force to "0" to get that step green
       # whilst still gathering data to the perf dashboards.
       # The result_type is used by the test_dispatcher to retry the test.
       exit_code = 0
 
     persisted_result = {
         'name': test_name,
@@ skipping 18 matching lines @@
     Returns:
       A tuple of (TestRunResults, retry).
     """
     _, result_type = self._LaunchPerfTest(test_name)
     results = base_test_result.TestRunResults()
     results.AddResult(base_test_result.BaseTestResult(test_name, result_type))
     retry = None
     if not results.DidRunPass():
       retry = test_name
     return results, retry