[Android] Ensure perf test runner does not create shards for blacklisted devices.

build/android/pylib/base/test_dispatcher.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Dispatches tests, either sharding or replicating them.
 
 Performs the following steps:
 * Create a test collection factory, using the given tests
   - If sharding: test collection factory returns the same shared test collection
     to all test runners
   - If replciating: test collection factory returns a unique test collection to
     each test runner, with the same set of tests in each.
 * Create a test runner for each device.
 * Run each test runner in its own thread, grabbing tests from the test
   collection until there are no tests left.
 """
 
 # TODO(jbudorick) Deprecate and remove this class after any relevant parts have
 # been ported to the new environment / test instance model.
 
 import logging
 import threading
 
+from devil.android import device_blacklist
 from devil.android import device_errors
 from devil.utils import reraiser_thread
 from devil.utils import watchdog_timer
 from pylib import constants
 from pylib.base import base_test_result
 from pylib.base import test_collection
 
 
 DEFAULT_TIMEOUT = 7 * 60  # seven minutes
 
@@ skipping 89 matching lines @@
     except:
       # An unhandleable exception, ensure tests get run by another device and
       # reraise this exception on the main thread.
       collection.add(test)
       raise
     finally:
       # Retries count as separate tasks so always mark the popped test as done.
       collection.test_completed()
 
 
-def _SetUp(runner_factory, device, out_runners, threadsafe_counter):
+def _SetUp(runner_factory, device, out_runners, threadsafe_counter,
+           blacklist_file=None):
   """Creates a test runner for each device and calls SetUp() in parallel.
 
   Note: if a device is unresponsive the corresponding TestRunner will not be
     added to out_runners.
 
   Args:
     runner_factory: Callable that takes a device and index and returns a
       TestRunner object.
     device: The device serial number to set up.
     out_runners: List to add the successfully set up TestRunner object.
     threadsafe_counter: A _ThreadSafeCounter object used to get shard indices.
+    blacklist_file: Path to device blacklist.
   """
   try:
     index = threadsafe_counter.GetAndIncrement()
     logging.warning('Creating shard %s for device %s.', index, device)
+
+    blacklist = (device_blacklist.Blacklist(blacklist_file).Read()

[jbudorick, 2016/06/06 22:52:27]

This isn't the right place to do this; it'll get read on all threads. This
should instead be handled in pylib.perf.setup.Setup, inside the
TestRunnerFactory it returns. I think you can do it by checking if the device is
in active_devices.

[rnephew (Reviews Here), 2016/06/06 23:33:21]

Yeah, that makes for a much much simpler solution.

+                 if blacklist_file
+                 else None)
+    if blacklist and str(device) in blacklist:
+      logging.info('Device %s is in blacklist. Not creating shard %s',
+                   str(device), index)
+      return
+
     runner = runner_factory(device, index)
     runner.SetUp()
     out_runners.append(runner)
   except (device_errors.CommandFailedError,
           device_errors.CommandTimeoutError,
           device_errors.DeviceUnreachableError):
     logging.exception('Failed to create shard for %s', str(device))
 
 
 def _RunAllTests(runners, test_collection_factory, num_retries, timeout=None,
@@ skipping 48 matching lines @@
       run_results.AddResults(base_test_result.BaseTestResult(
           t, base_test_result.ResultType.UNKNOWN) for t in tc.test_names())
 
   for r in results:
     run_results.AddTestRunResults(r)
   if not run_results.DidRunPass():
     exit_code = constants.ERROR_EXIT_CODE
   return (run_results, exit_code)
 
 
-def _CreateRunners(runner_factory, devices, timeout=None):
+def _CreateRunners(runner_factory, devices, timeout=None, blacklist_file=None):
   """Creates a test runner for each device and calls SetUp() in parallel.
 
   Note: if a device is unresponsive the corresponding TestRunner will not be
     included in the returned list.
 
   Args:
     runner_factory: Callable that takes a device and index and returns a
       TestRunner object.
     devices: List of device serial numbers as strings.
     timeout: Watchdog timeout in seconds, defaults to the default timeout.
+    blacklist_file: Path to device blacklist file.
 
   Returns:
     A list of TestRunner objects.
   """
   logging.warning('Creating %s test %s.', len(devices),
                   'runners' if len(devices) != 1 else 'runner')
   runners = []
   counter = _ThreadSafeCounter()
   threads = reraiser_thread.ReraiserThreadGroup(
       [reraiser_thread.ReraiserThread(_SetUp,
-                                      [runner_factory, d, runners, counter],
+                                      [runner_factory, d, runners, counter,
+                                       blacklist_file],
                                       name=str(d)[-4:])
        for d in devices])
   threads.StartAll()
   threads.JoinAll(watchdog_timer.WatchdogTimer(timeout))
   return runners
 
 
 def _TearDownRunners(runners, timeout=None):
   """Calls TearDown() for each test runner in parallel.
 
@@ skipping 25 matching lines @@
       tests_expanded.append(test_group)
     else:
       test_split = test_group.split(':')
       for i in range(0, len(test_split), max_per_run):
         tests_expanded.append(':'.join(test_split[i:i+max_per_run]))
   return tests_expanded
 
 
 def RunTests(tests, runner_factory, devices, shard=True,
              test_timeout=DEFAULT_TIMEOUT, setup_timeout=DEFAULT_TIMEOUT,
-             num_retries=2, max_per_run=256):
+             num_retries=2, max_per_run=256, blacklist_file=None):
   """Run all tests on attached devices, retrying tests that don't pass.
 
   Args:
     tests: List of tests to run.
     runner_factory: Callable that takes a device and index and returns a
         TestRunner object.
     devices: List of attached devices.
     shard: True if we should shard, False if we should replicate tests.
       - Sharding tests will distribute tests across all test runners through a
         shared test collection.
       - Replicating tests will copy all tests to each test runner through a
         unique test collection for each test runner.
     test_timeout: Watchdog timeout in seconds for running tests.
     setup_timeout: Watchdog timeout in seconds for creating and cleaning up
         test runners.
     num_retries: Number of retries for a test.
     max_per_run: Maximum number of tests to run in any group.
+    blacklist_file: Path to blacklist file.
 
   Returns:
     A tuple of (base_test_result.TestRunResults object, exit code).
   """
   if not tests:
     logging.critical('No tests to run.')
     return (base_test_result.TestRunResults(), constants.ERROR_EXIT_CODE)
 
   tests_expanded = ApplyMaxPerRun(tests, max_per_run)
   if shard:
     # Generate a shared TestCollection object for all test runners, so they
     # draw from a common pool of tests.
     shared_test_collection = test_collection.TestCollection(
         [_Test(t) for t in tests_expanded])
     test_collection_factory = lambda: shared_test_collection
     tag_results_with_device = False
     log_string = 'sharded across devices'
   else:
     # Generate a unique TestCollection object for each test runner, but use
     # the same set of tests.
     test_collection_factory = lambda: test_collection.TestCollection(
         [_Test(t) for t in tests_expanded])
     tag_results_with_device = True
     log_string = 'replicated on each device'
 
   logging.info('Will run %d tests (%s): %s',
                len(tests_expanded), log_string, str(tests_expanded))
-  runners = _CreateRunners(runner_factory, devices, setup_timeout)
+  runners = _CreateRunners(runner_factory, devices, setup_timeout,
+                           blacklist_file=blacklist_file)
   try:
     return _RunAllTests(runners, test_collection_factory,
                         num_retries, test_timeout, tag_results_with_device)
   finally:
     try:
       _TearDownRunners(runners, setup_timeout)
     except device_errors.DeviceUnreachableError as e:
       logging.warning('Device unresponsive during TearDown: [%s]', e)
     except Exception: # pylint: disable=broad-except
       logging.exception('Unexpected exception caught during TearDown')