[Android] Re-write the gtest TestRunner and introduce a new generic sharder.

build/android/pylib/base/shard.py

Index: build/android/pylib/base/shard.py
diff --git a/build/android/pylib/base/shard.py b/build/android/pylib/base/shard.py
new file mode 100644
index 0000000000000000000000000000000000000000..a0a4f7f3dd5a5831117785c8185678e5f316655b
--- /dev/null
+++ b/build/android/pylib/base/shard.py
@@ -0,0 +1,154 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Implements test sharding logic."""
+
+import logging
+import sys
+import threading
+
+from pylib import android_commands
+from pylib import forwarder
+
+import test_result
+
+
+class _Worker(threading.Thread):
+  """Runs tests from the test_queue using the given runner in a separate thread.
+
+  Places results in the results_list.

[frankf, 2013/02/20 18:55:31]

results_list -> out_results

[craigdh, 2013/02/20 19:38:09]

Done.

+  """
+  def __init__(self, runner, test_queue, out_results, out_retry):
+    """Initializes the worker.
+
+    Args:
+      runner: A TestRunner object used to run the tests.
+      test_queue: A list from which to get tests to run.
+      out_results: A list to add TestResults to.
+      out_retry: A list to add tests to retry.
+    """
+    super(_Worker, self).__init__()
+    self.daemon = True
+    self._exc_info = None
+    self._runner = runner
+    self._test_queue = test_queue
+    self._out_results = out_results
+    self._out_retry = out_retry
+
+  def run(self):
+    """Run tests from the queue until it is empty, storing results.

[frankf, 2013/02/20 18:55:31]

be more precise than "storing results" :)

[craigdh, 2013/02/20 19:38:09]

Done.

+
+    Overriden from threading.Thread, runs in a separate thread.

[frankf, 2013/02/20 18:55:31]

I think we do something like #Override before the method.

[craigdh, 2013/02/20 19:38:09]

Done.

+    """
+    try:
+      while True:
+        test = self._test_queue.pop()
+        result, retry = self._runner.Run(test)
+        self._out_results.append(result)
+        if retry:
+          self._out_retry.append(retry)
+    except IndexError:
+      pass
+    except:
+      self._exc_info = sys.exc_info()
+      raise
+
+  def Reraise(self):
+    """Reraise an exception raised in the thread."""

[frankf, 2013/02/20 18:55:31]

Reraise _if_ an exception...

Maybe rename to better reflect the conditional

[craigdh, 2013/02/20 19:38:09]

Done.

+    if self._exc_info:
+      raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
+
+
+def _RunAllTests(runners, tests):
+  """Run all tests using the given TestRunners.
+
+  Args:
+    runners: a list of TestRunner objects.
+    tests: a list of Tests to run using the given TestRunners.
+
+  Returns:
+    Tuple: (list of TestResults, list of tests to retry)
+  """
+  tests_queue = list(tests)
+  workers = []
+  results = []
+  retry = []
+  for r in runners:
+    worker = _Worker(r, tests_queue, results, retry)
+    worker.start()
+    workers.append(worker)
+  while workers:
+    for w in workers[:]:
+      # Allow the main thread to periodically check for keyboard interrupts.
+      w.join(0.1)
+      if not w.isAlive():
+        w.Reraise()
+        workers.remove(w)
+  return (results, retry)
+
+
+def _CreateRunners(runner_factory, devices):
+  """Creates a test runner for each device.
+
+  Note: if a device is unresponsive the corresponding TestRunner will not be
+    included in the returned list.
+
+  Args:
+    runner_factory: callable that takes a device and returns a TestRunner.
+    devices: list of device serial numbers as strings.
+
+  Returns:
+    A list of TestRunner objects.
+  """
+  test_runners = []
+  for index, device in enumerate(devices):
+    logging.warning('*' * 80)
+    logging.warning('Creating shard %d for %s', index, device)
+    logging.warning('*' * 80)
+    try:
+      test_runners.append(runner_factory(device))
+    except android_commands.errors.DeviceUnresponsiveError as e:
+      logging.warning('****Failed to create a shard: [%s]', e)

[frankf, 2013/02/20 18:55:31]

Does this print the device that failed?

[craigdh, 2013/02/20 19:38:09]

I'm not sure if the exception message includes the device or just the reason for
the failure, but if not the device is listed two lines above where it says
'Creating shard NUM for DEVICE.'

+  return test_runners
+
+
+def ShardAndRunTests(
+    runner_factory, devices, tests, build_type='Debug', tries=3,
+    get_attached_devices_callable=android_commands.GetAttachedDevices):
+  """Run all tests on attached devices, retrying tests that don't pass.
+
+  Args:
+    runner_factory: callable that takes a device and returns a TestRunner.
+    devices: list of attached device serial numbers as strings.
+    tests: list of tests to run.
+    build_type: either 'Debug' or 'Release'.
+    retries: number of retries before accepting failure.

[frankf, 2013/02/20 18:55:31]

update this.

[craigdh, 2013/02/20 19:38:09]

Done.

+    get_attached_devices_callable: only override default value for testing.

[frankf, 2013/02/20 18:55:31]

Can you use pymox to mock out the calls to adb instead?

[craigdh, 2013/02/20 19:38:09]

Removed this, it was a trivial one-liner to mock out in the unittest. PyMox
looks neat, but it may be overkill here. If we find ourselves requring pymox
later on for more involved mocking it will be trivial to switch this out, but
for now I'd rather not put another road block between people
running/contributing unittests (as pymox has to be installed from source).

+
+  Returns:
+    A test_result.TestResults object.
+  """
+  final_results = test_result.TestResults()
+  results = test_result.TestResults()
+  forwarder.Forwarder.KillHost(build_type)
+  try_count = 0
+  while tests:
+    devices = set(devices).intersection(get_attached_devices_callable())
+    if not devices:
+      logging.critical('No devices attached and visible to run tests!')

[frankf, 2013/02/20 18:55:31]

msg = ...

[craigdh, 2013/02/20 19:38:09]

Done.

+      raise Exception('No devices attached and visible to run tests!')
+    if try_count >= tries:

[frankf, 2013/02/20 18:55:31]

Add some comments for these termination conditions 

[craigdh, 2013/02/20 19:38:09]

Done.

+      results.ok = final_results.ok
+      final_results = results
+      break
+    try_count += 1
+    runners = _CreateRunners(runner_factory, devices)
+    try:
+      results_list, tests = _RunAllTests(runners, tests)
+      results = test_result.TestResults.FromTestResults(results_list)
+      final_results.ok += results.ok
+    except android_commands.errors.DeviceUnresponsiveError as e:
+      logging.warning('****Failed to run test: [%s]', e)
+  forwarder.Forwarder.KillHost(build_type)
+  return final_results