[Android] Re-write the gtest TestRunner and introduce a new generic sharder.

build/android/pylib/base/shard.py

Index: build/android/pylib/base/shard.py
diff --git a/build/android/pylib/base/shard.py b/build/android/pylib/base/shard.py
new file mode 100644
index 0000000000000000000000000000000000000000..f47352aeafe1389dc9d741347bb9665572cdb8ad
--- /dev/null
+++ b/build/android/pylib/base/shard.py
@@ -0,0 +1,149 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Implements test sharding logic."""
+
+import logging
+import sys
+import threading
+
+from pylib import android_commands
+from pylib import forwarder
+
+import test_result
+
+
+class _Worker(threading.Thread):
+  """Runs tests from the test_queue using the given runner in a separate thread.
+
+  Places results in the results_list.
+  """
+  def __init__(self, runner, test_queue, out_results, out_retry):
+    """Initializes the worker.
+
+    Args:
+      runner: A TestRunner object used to run the tests.
+      test_queue: A list from which to get tests to run.
+      out_results: A list to add TestResults to.
+      out_retry: A list to add tests to retry.
+    """
+    super(_Worker, self).__init__()
+    self.daemon = True
+    self._exc_info = None
+    self._runner = runner
+    self._test_queue = test_queue
+    self._out_results = out_results
+    self._out_retry = out_retry
+
+  def run(self):
+    """Run tests from the queue until it is empty, storing results.
+
+    Overriden from threading.Thread, runs in a separate thread.
+    """
+    try:
+      while True:
+        test = self._test_queue.pop()
+        result, retry = self._runner.Run(test)
+        self._out_results.append(result)
+        if retry:
+          self._out_retry.append(retry)
+    except IndexError:
+      pass
+    except:
+      self._exc_info = sys.exc_info()
+      raise
+
+  def Reraise(self):
+    """Reraise an exception raised in the thread."""
+    if self._exc_info:
+      raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
+
+
+def _RunAllTests(runners, tests):
+  """Run all tests using the given TestRunners.
+
+  Args:
+    runners: a list of TestRunner objects.
+    tests: a list of Tests to run using the given TestRunners.
+
+  Returns:
+    Tuple: (list of TestResults, list of tests to retry)
+  """
+  tests_queue = list(tests)
+  workers = []
+  results = []
+  retry = []
+  for r in runners:
+    worker = _Worker(r, tests_queue, results, retry)
+    worker.start()
+    workers.append(worker)
+  while workers:
+    for w in workers[:]:
+      # Allow the main thread to periodically check for keyboard interrupts.
+      w.join(0.1)
+      if not w.isAlive():
+        w.Reraise()
+        workers.remove(w)
+  return (results, retry)
+
+
+def _CreateRunners(runner_factory, devices):
+  """Creates a test runner for each device.

[frankf, 2013/02/19 23:57:26]

Expand to include throwing out devices.

[craigdh, 2013/02/20 00:05:25]

Done.

+
+  Args:
+    runner_factory: callable that takes a device and returns a TestRunner.
+    devices: list of device serial numbers as strings.
+
+  Returns:
+    A list of TestRunner objects.
+  """
+  test_runners = []
+  for index, device in enumerate(devices):
+    logging.warning('*' * 80)
+    logging.warning('Creating shard %d for %s', index, device)
+    logging.warning('*' * 80)
+    try:
+      test_runners.append(runner_factory(device))
+    except android_commands.errors.DeviceUnresponsiveError as e:
+      logging.critical('****Failed to create a shard: [%s]', e)

[frankf, 2013/02/19 23:57:26]

This is warning not critical.

[craigdh, 2013/02/20 00:05:25]

Done.

+  return test_runners
+
+
+def ShardAndRunTests(
+    runner_factory, devices, tests, build_type='Debug', retries=3,

[frankf, 2013/02/19 23:57:26]

retries -> tries to avoid off-by-1.

[craigdh, 2013/02/20 00:05:25]

Done.

+    get_attached_devices_callable=android_commands.GetAttachedDevices):
+  """Run all tests on attached devices, retrying tests that don't pass.
+
+  Args:
+    runner_factory: callable that takes a device and returns a TestRunner.
+    devices: list of attached device serial numbers as strings.
+    tests: list of tests to run.
+    build_type: either 'Debug' or 'Release'.
+    retries: number of retries before accepting failure.
+    get_attached_devices_callable: only override default value for testing.
+
+  Returns:
+    A test_result.TestResults object.
+  """
+  final_results = test_result.TestResults()
+  results = test_result.TestResults()
+  forwarder.Forwarder.KillHost(build_type)
+  tries = 0
+  while tests:
+    devices = set(devices).intersection(get_attached_devices_callable())
+    if not devices or tries > retries:
+      results.ok = final_results.ok
+      final_results = results
+      break
+    tries += 1
+    runners = _CreateRunners(runner_factory, devices)
+    try:
+      results_list, tests = _RunAllTests(runners, tests)
+      results = test_result.TestResults.FromTestResults(results_list)
+    except android_commands.errors.DeviceUnresponsiveError as e:
+      logging.critical('****Failed to run test: [%s]', e)

[frankf, 2013/02/19 23:57:26]

warning not critical.

[craigdh, 2013/02/20 00:05:25]

Done.

+      continue
+    final_results.ok += results.ok

[frankf, 2013/02/19 23:57:26]

Move to end of try.

[craigdh, 2013/02/20 00:05:25]

Done.

+  forwarder.Forwarder.KillHost(build_type)
+  return final_results