Allow test_runner.py to run host perf tests without devices attached

build/android/pylib/local/device/local_device_perf_test_run.py

Index: build/android/pylib/local/device/local_device_perf_test_run.py
diff --git a/build/android/pylib/local/device/local_device_perf_test_run.py b/build/android/pylib/local/device/local_device_perf_test_run.py
index b78553e0417f6825fa0c5d981fb454dda26e0123..97c2360b97c0be9d839a952a1b5ca9e3be1185be 100644
--- a/build/android/pylib/local/device/local_device_perf_test_run.py
+++ b/build/android/pylib/local/device/local_device_perf_test_run.py
@@ -22,6 +22,7 @@ from devil.android.tools import device_recovery
 from devil.android.tools import device_status
 from devil.utils import cmd_helper
 from devil.utils import parallelizer
+from devil.utils import reraiser_thread
 from pylib import constants
 from pylib.base import base_test_result
 from pylib.constants import host_paths
@@ -299,7 +300,7 @@ class HostTestShard(TestShard):
   def RunTestsOnShard(self):
     results = base_test_result.TestRunResults()
     for test in self._tests:
-      tries_left = self._retries
+      tries_left = self._retries + 1
       result_type = None
       while (result_type != base_test_result.ResultType.PASS
              and tries_left > 0):
@@ -334,9 +335,6 @@ class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun):
     self._timeout = None if test_instance.no_timeout else self._DEFAULT_TIMEOUT
 
   def SetUp(self):
-    self._devices = self._GetAllDevices(self._env.devices,
-                                        self._test_instance.known_devices_file)
-
     if os.path.exists(constants.PERF_OUTPUT_DIR):
       shutil.rmtree(constants.PERF_OUTPUT_DIR)
     os.makedirs(constants.PERF_OUTPUT_DIR)
@@ -412,27 +410,39 @@ class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun):
     if not self._test_buckets and not self._no_device_tests:
       raise local_device_test_run.NoTestsError()
 
-    def run_perf_tests(shard_id):
-      if shard_id is None:
-        s = HostTestShard(self._env, self._test_instance, self._no_device_tests,
-                          retries=3, timeout=self._timeout)
-      else:
-        if device_status.IsBlacklisted(
-             str(self._devices[shard_id]), self._env.blacklist):
-          logging.warning('Device %s is not active. Will not create shard %s.',
-                          str(self._devices[shard_id]), shard_id)
-          return None
-        s = DeviceTestShard(self._env, self._test_instance,
-                            self._devices[shard_id], shard_id,
-                            self._test_buckets[shard_id],
-                            retries=self._env.max_tries, timeout=self._timeout)
+    def run_no_devices_tests():
+      if not self._no_device_tests:
+        return []
+      s = HostTestShard(self._env, self._test_instance, self._no_device_tests,
+                        retries=0, timeout=self._timeout)

[jbudorick, 2016/09/08 02:45:59]

nit: I'm fine with dropping retries, but I'd do it in a separate CL.

[agrieve, 2016/09/08 02:57:19]

Done.

+      return [s.RunTestsOnShard()]
+
+    def shard_helper(shard_id):

[jbudorick, 2016/09/08 02:45:59]

nit: maybe device_shard_helper

+      if device_status.IsBlacklisted(
+           str(self._devices[shard_id]), self._env.blacklist):
+        logging.warning('Device %s is not active. Will not create shard %s.',
+                        str(self._devices[shard_id]), shard_id)
+        return None
+      s = DeviceTestShard(self._env, self._test_instance,
+                          self._devices[shard_id], shard_id,
+                          self._test_buckets[shard_id],
+                          retries=self._env.max_tries, timeout=self._timeout)
       return s.RunTestsOnShard()
 
-    device_indices = range(min(len(self._devices), len(self._test_buckets)))
-    if self._no_device_tests:
-      device_indices.append(None)
-    shards = parallelizer.Parallelizer(device_indices).pMap(run_perf_tests)
-    return [x for x in shards.pGet(self._timeout) if x is not None]
+    def run_devices_tests():
+      if not self._test_buckets:
+        return []
+      if self._devices is None:
+        self._devices = self._GetAllDevices(
+            self._env.devices, self._test_instance.known_devices_file)
+
+      device_indices = range(min(len(self._devices), len(self._test_buckets)))
+      shards = parallelizer.Parallelizer(device_indices).pMap(shard_helper)
+      return [x for x in shards.pGet(self._timeout) if x is not None]
+
+    host_test_results, device_test_results = reraiser_thread.RunAsync(

[jbudorick, 2016/09/08 02:45:59]

I'm wondering if there's a clean way to do shard creation separately by
host/device, then run the shards all together w/ the parallelizer and something
like

  def run_shard(shard):
    return s.RunTestsOnShard()

  shards = # logic to create host and device shards as necessary
  running_shards = parallelizer.Parallelizer(shards).pMap(run_shard)
  return [x for x in running_shards.pGet(self._timeout) if x is not None]

[agrieve, 2016/09/08 02:57:19]

I think this could work so long as we still are careful to not query for devices
when self._test_buckets == []. Okay to do it in a follow-up? I won't have time
to figure that out tonight (I'm not sure I'm grokking the blacklisting part of
this quite yet).

[jbudorick, 2016/09/08 02:59:51]

Yeah, I'm fine with doing it in a follow-up. This version certainly has fewer
moving parts than that one.

+        [run_no_devices_tests, run_devices_tests])
+    return host_test_results + device_test_results
 
   # override
   def TestPackage(self):