[Android] Re-write the gtest TestRunner and introduce a new generic sharder.

build/android/pylib/shard.py

+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Implements test sharding logic."""
+
+import logging
+import Queue
+import sys
+import threading
+
+from pylib import android_commands
+from pylib.base import test_result
+from pylib import forwarder
+
+
+class _Worker(threading.Thread):
+  """Runs tests from the test_queue using the given runner in a separate thread.
+
+  Places results in the results_list.
+  """
+  def __init__(self, runner, test_queue, out_results, out_retry):
+    """Initializes the worker.
+
+    Args:
+      runner: A TestRunner object used to run the tests.
+      test_queue: A Queue.Queue from which to get Tests to run.
+      out_results: A list to add TestResults to.
+      out_retry: A list to add tests to retry.
+    """
+    super(_Worker, self).__init__()
+    self.daemon = True
+    self._exc_info = None
+    self._runner = runner
+    self._test_queue = test_queue
+    self._out_results = out_results
+    self._out_retry = out_retry
+
+  def run(self):
+    """Run tests from the queue until it is empty, storing results.
+
+    Overriden from threading.Thread, runs in a separate thread.
+    """
+    try:
+      while True:
+        test = self._test_queue.get(block=False)
+        result, retry = self._runner.Run(test)
+        self._out_results.append(result)
+        if retry:
+          self._out_retry.append(retry)
+    except Queue.Empty:
+      pass
+    except:
+      self._exc_info = sys.exc_info()
+      raise
+
+  def Reraise(self):
+    """Reraise an exception raised in the thread."""
+    if self._exc_info:
+      raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
+
+
+def _RunAllTests(runners, tests):
+  """Run all tests using the given TestRunners.
+
+  Args:
+    runners: a list of TestRunner objects.
+    tests: a list of Tests to run using the given TestRunners.
+
+  Returns:
+    Tuple: (list of TestResults, list of tests to retry)
+  """
+  tests_queue = Queue.Queue()
+  for t in tests:
+    tests_queue.put(t)
+  workers = []
+  results = []
+  retry = []
+  for r in runners:
+    worker = _Worker(r, tests_queue, results, retry)
+    worker.start()
+    workers.append(worker)
+  for w in workers:
+    w.join()
+    w.Reraise()
+  return (results, retry)
+
+
+def _CreateRunners(runner_factory, devices):
+  """Creates a test runner for each device.
+
+  Args:
+    runner_factory: callable that takes a device and returns a TestRunner.
+    devices: list of device serial numbers as strings.
+  """
+  test_runners = []
+  for index, device in enumerate(devices):
+    logging.warning('*' * 80)
+    logging.warning('Creating shard %d for %s', index, device)
+    logging.warning('*' * 80)
+    try:
+      test_runners.append(runner_factory(device))
+    except android_commands.errors.DeviceUnresponsiveError as e:

[frankf, 2013/02/19 22:48:50]

Is there a point to this except, we catch DeviceUnresponsiveError on line 132 as
well.

[craigdh, 2013/02/19 23:38:30]

This is called before the try block that line 132 catches. Essentially we just
skip creating a TestRunner for any device that fails. Line 132 is for failures
that occur while running a test.

+      logging.critical('****Failed to create a shard: [%s]', e)
+  return test_runners
+
+
+def Shard(runner_factory, devices, tests, build_type='Debug'):
+  """Run all tests on attached devices, retrying tests that don't pass.
+
+  Args:
+    runner_factory: callable that takes a device and returns a TestRunner.
+    devices: list of attached device serial numbers as strings.
+    tests: list of Test objects to run.

[frankf, 2013/02/19 22:48:50]

tests is just a list of strings.

[craigdh, 2013/02/19 23:38:30]

Good catch. Originally it wasn't going to be. Fixed.

+    build_type: either 'Debug' or 'Release'.

[frankf, 2013/02/19 22:48:50]

Side note: This is another global config that is trickled down into every
function.

+  """
+  final_results = test_result.TestResults()
+  forwarder.Forwarder.KillHost(build_type)
+  retries = 3
+  tries = 0
+  while tests:
+    devices = set(devices).intersection(android_commands.GetAttachedDevices())
+    if not devices or tries > retries:

[frankf, 2013/02/19 22:48:50]

tries >= retires?

[craigdh, 2013/02/19 23:38:30]

Depends, should retries count the first try or just retries?

+      results.ok = final_results.ok
+      final_results = results
+      break
+    tries += 1
+    runners = _CreateRunners(runner_factory, devices)
+    try:
+      results_list, tests = _RunAllTests(runners, tests)
+      results = test_result.TestResults.FromTestResults(results_list)
+    except android_commands.errors.DeviceUnresponsiveError as e:
+      logging.critical('****Failed to run test: [%s]', e)
+      continue
+    final_results.ok += results.ok
+  forwarder.Forwarder.KillHost(build_type)
+  return final_results