[Android] Add ability to run deviceless tests on hosts without devices for perf tests.

build/android/pylib/local/device/local_device_perf_test_run.py

Index: build/android/pylib/local/device/local_device_perf_test_run.py
diff --git a/build/android/pylib/local/device/local_device_perf_test_run.py b/build/android/pylib/local/device/local_device_perf_test_run.py
index d6bf36899062bc77350fe1c87e71ae13de526613..9596be4928a2c62c7434d219186105846f9c534a 100644
--- a/build/android/pylib/local/device/local_device_perf_test_run.py
+++ b/build/android/pylib/local/device/local_device_perf_test_run.py
@@ -61,78 +61,20 @@ class HeartBeat(object):
 
 
 class TestShard(object):
-  def __init__(
-      self, env, test_instance, device, index, tests, retries=3, timeout=None):
-    logging.info('Create shard %s for device %s to run the following tests:',
-                 index, device)
+  def __init__(self, env, test_instance, tests, retries=3, timeout=None):
+    logging.info('Create host shard for the following tests:')

[jbudorick, 2016/08/02 18:51:50]

This shouldn't be saying it's a host shard.

[rnephew (Reviews Here), 2016/08/02 19:03:52]

Done.

     for t in tests:
       logging.info('  %s', t)
-    self._battery = battery_utils.BatteryUtils(device)
     self._current_test = None
-    self._device = device
+    self._device = None

[jbudorick, 2016/08/02 18:51:50]

Why is this still here rather than in DeviceTestShard?

[rnephew (Reviews Here), 2016/08/02 19:03:52]

The CreateCmd function needs it. I decided to put it in here because I didn't
want both other test runners to have nearly identical methods with only one line
of code difference.

     self._env = env
-    self._index = index
+    self._heart_beat = HeartBeat(self)
+    self._index = -1
     self._output_dir = None
     self._retries = retries
     self._test_instance = test_instance
     self._tests = tests
     self._timeout = timeout
-    self._heart_beat = HeartBeat(self)
-
-  @local_device_environment.handle_shard_failures
-  def RunTestsOnShard(self):
-    results = base_test_result.TestRunResults()
-    for test in self._tests:
-      tries_left = self._retries
-      result_type = None
-      while (result_type != base_test_result.ResultType.PASS
-             and tries_left > 0):
-        try:
-          self._TestSetUp(test)
-          result_type = self._RunSingleTest(test)
-        except device_errors.CommandTimeoutError:
-          result_type = base_test_result.ResultType.TIMEOUT
-        except device_errors.CommandFailedError:
-          logging.exception('Exception when executing %s.', test)
-          result_type = base_test_result.ResultType.FAIL
-        finally:
-          self._TestTearDown()
-          if result_type != base_test_result.ResultType.PASS:
-            try:
-              device_recovery.RecoverDevice(self._device, self._env.blacklist)
-            except device_errors.CommandTimeoutError:
-              logging.exception(
-                  'Device failed to recover after failing %s.', test)
-          tries_left = tries_left - 1
-
-      results.AddResult(base_test_result.BaseTestResult(test, result_type))
-    return results
-
-  def _TestSetUp(self, test):
-    if not self._device.IsOnline():
-      msg = 'Device %s is unresponsive.' % str(self._device)
-      raise device_errors.DeviceUnreachableError(msg)
-
-    logging.info('Charge level: %s%%',
-                 str(self._battery.GetBatteryInfo().get('level')))
-    if self._test_instance.min_battery_level:
-      self._battery.ChargeDeviceToLevel(self._test_instance.min_battery_level)
-
-    logging.info('temperature: %s (0.1 C)',
-                 str(self._battery.GetBatteryInfo().get('temperature')))
-    if self._test_instance.max_battery_temp:
-      self._battery.LetBatteryCoolToTemperature(
-          self._test_instance.max_battery_temp)
-
-    if not self._device.IsScreenOn():
-      self._device.SetScreen(True)
-
-    if (self._test_instance.collect_chartjson_data
-        or self._tests[test].get('archive_output_dir')):
-      self._output_dir = tempfile.mkdtemp()
-
-    self._current_test = test
-    self._heart_beat.Start()
 
   def _RunSingleTest(self, test):
     self._test_instance.WriteBuildBotJson(self._output_dir)
@@ -165,7 +107,9 @@ class TestShard(object):
                                    output, json_output, result_type)
 
   def _CreateCmd(self, test):
-    cmd = '%s --device %s' % (self._tests[test]['cmd'], str(self._device))
+    cmd = '%s' % self._tests[test]['cmd']
+    if self._device:
+      cmd = '%s --device %s' % (cmd, str(self._device))
     if self._output_dir:
       cmd = cmd + ' --output-dir=%s' % self._output_dir
     if self._test_instance.dry_run:
@@ -233,6 +177,75 @@ class TestShard(object):
     with file(pickled, 'w') as f:
       f.write(pickle.dumps(result))
 
+  @property
+  def current_test(self):
+    return self._current_test
+
+
+class DeviceTestShard(TestShard):
+  def __init__(
+      self, env, test_instance, device, index, tests, retries=3, timeout=None):
+    super(DeviceTestShard, self).__init__(
+        env, test_instance, tests, retries, timeout)
+    self._battery = battery_utils.BatteryUtils(device) if device else None
+    self._device = device
+    self._index = index
+
+  @local_device_environment.handle_shard_failures
+  def RunTestsOnShard(self):
+    results = base_test_result.TestRunResults()
+    for test in self._tests:
+      tries_left = self._retries
+      result_type = None
+      while (result_type != base_test_result.ResultType.PASS
+             and tries_left > 0):
+        try:
+          self._TestSetUp(test)
+          result_type = self._RunSingleTest(test)
+        except device_errors.CommandTimeoutError:
+          result_type = base_test_result.ResultType.TIMEOUT
+        except device_errors.CommandFailedError:
+          logging.exception('Exception when executing %s.', test)
+          result_type = base_test_result.ResultType.FAIL
+        finally:
+          self._TestTearDown()
+          if result_type != base_test_result.ResultType.PASS:
+            try:
+              device_recovery.RecoverDevice(self._device, self._env.blacklist)
+            except device_errors.CommandTimeoutError:
+              logging.exception(
+                  'Device failed to recover after failing %s.', test)
+          tries_left = tries_left - 1
+
+      results.AddResult(base_test_result.BaseTestResult(test, result_type))
+    return results
+
+  def _TestSetUp(self, test):
+    if not self._device.IsOnline():
+      msg = 'Device %s is unresponsive.' % str(self._device)
+      raise device_errors.DeviceUnreachableError(msg)
+
+    logging.info('Charge level: %s%%',
+                 str(self._battery.GetBatteryInfo().get('level')))
+    if self._test_instance.min_battery_level:
+      self._battery.ChargeDeviceToLevel(self._test_instance.min_battery_level)
+
+    logging.info('temperature: %s (0.1 C)',
+                 str(self._battery.GetBatteryInfo().get('temperature')))
+    if self._test_instance.max_battery_temp:
+      self._battery.LetBatteryCoolToTemperature(
+          self._test_instance.max_battery_temp)
+
+    if not self._device.IsScreenOn():
+      self._device.SetScreen(True)
+
+    if (self._test_instance.collect_chartjson_data
+        or self._tests[test].get('archive_output_dir')):
+      self._output_dir = tempfile.mkdtemp()
+
+    self._current_test = test
+    self._heart_beat.Start()
+
   def _TestTearDown(self):
     if self._output_dir:
       shutil.rmtree(self._output_dir, ignore_errors=True)
@@ -246,9 +259,42 @@ class TestShard(object):
       self._heart_beat.Stop()
       self._current_test = None
 
-  @property
-  def current_test(self):
-    return self._current_test
+
+class HostTestShard(TestShard):
+  def __init__(self, env, test_instance, tests, retries=3, timeout=None):
+    super(HostTestShard, self).__init__(
+        env, test_instance, tests, retries, timeout)
+
+  @local_device_environment.handle_shard_failures
+  def RunTestsOnShard(self):
+    results = base_test_result.TestRunResults()
+    for test in self._tests:
+      tries_left = self._retries
+      result_type = None
+      while (result_type != base_test_result.ResultType.PASS
+             and tries_left > 0):
+        try:
+          self._TestSetUp(test)
+          result_type = self._RunSingleTest(test)
+        finally:
+          self._TestTearDown()
+      results.AddResult(base_test_result.BaseTestResult(test, result_type))
+    return results
+
+  def _TestSetUp(self, test):
+    if (self._test_instance.collect_chartjson_data
+        or self._tests[test].get('archive_output_dir')):
+      self._output_dir = tempfile.mkdtemp()
+    self._current_test = test
+    self._heart_beat.Start()
+
+  def _TestTearDown(self):
+    if self._output_dir:
+      shutil.rmtree(self._output_dir, ignore_errors=True)
+      self._output_dir = None
+    self._heart_beat.Stop()
+    self._current_test = None
+
 
 class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun):
 
@@ -259,6 +305,7 @@ class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun):
     super(LocalDevicePerfTestRun, self).__init__(env, test_instance)
     self._devices = None
     self._env = env
+    self._no_device_tests = {}
     self._test_buckets = []
     self._test_instance = test_instance
     self._timeout = None if test_instance.no_timeout else self._DEFAULT_TIMEOUT
@@ -305,10 +352,13 @@ class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun):
     for test, test_config in test_dict['steps'].iteritems():
       try:
         affinity = test_config['device_affinity']
-        if len(self._test_buckets) < affinity + 1:
-          while len(self._test_buckets) != affinity + 1:
-            self._test_buckets.append({})
-        self._test_buckets[affinity][test] = test_config
+        if affinity == -1:

[jbudorick, 2016/08/02 18:51:50]

Could this allow for multiple host test shards by just handling negative numbers
as host shards?

[rnephew (Reviews Here), 2016/08/02 19:03:53]

Probably, but I'm not sure there is much advantage in allowing for more than one
thread doing on-host tests. There aren't many of them.

+          self._no_device_tests[test] = test_config
+        else:
+          if len(self._test_buckets) < affinity + 1:
+            while len(self._test_buckets) != affinity + 1:
+              self._test_buckets.append({})
+          self._test_buckets[affinity][test] = test_config
       except KeyError:
         logging.exception(
             'Test config for %s is bad.\n Config:%s', test, str(test_config))
@@ -336,21 +386,28 @@ class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun):
   def RunTests(self):
     # Affinitize the tests.
     self._SplitTestsByAffinity()
-    if not self._test_buckets:
+    if not self._test_buckets and not self._no_device_tests:
       raise local_device_test_run.NoTestsError()
 
     def run_perf_tests(shard_id):
+      if shard_id == -1:

[jbudorick, 2016/08/02 18:51:50]

nit: I'd prefer

  if shard_id == -1:
    s = HostTestShard(...)
  else:
    if device_status.IsBlacklisted(...):
      ...
    s = DeviceTestShard(...)
  return s.RunTestsOnShard()

[rnephew (Reviews Here), 2016/08/02 19:03:53]

Done.

+        s = HostTestShard(self._env, self._test_instance,
+                      self._no_device_tests, retries=3, timeout=self._timeout)

[jbudorick, 2016/08/02 18:51:50]

nit: indentation is off

[rnephew (Reviews Here), 2016/08/02 19:03:52]

Done.

+        return s.RunTestsOnShard()
       if device_status.IsBlacklisted(
           str(self._devices[shard_id]), self._env.blacklist):
         logging.warning('Device %s is not active. Will not create shard %s.',
                         str(self._devices[shard_id]), shard_id)
         return None
-      s = TestShard(self._env, self._test_instance, self._devices[shard_id],
-                    shard_id, self._test_buckets[shard_id],
-                    retries=self._env.max_tries, timeout=self._timeout)
+      s = DeviceTestShard(self._env, self._test_instance,
+                          self._devices[shard_id], shard_id,
+                          self._test_buckets[shard_id],
+                          retries=self._env.max_tries, timeout=self._timeout)
       return s.RunTestsOnShard()
 
     device_indices = range(min(len(self._devices), len(self._test_buckets)))
+    if self._no_device_tests:
+      device_indices.append(-1)
     shards = parallelizer.Parallelizer(device_indices).pMap(run_perf_tests)
     return [x for x in shards.pGet(self._timeout) if x is not None]