Android: adds device affinity for perf tests.

build/android/pylib/perf/setup.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generates test runner factory and tests for performance tests."""
 
 import json
 import fnmatch
+import logging
 import os
 import shutil
 
+from pylib import android_commands
 from pylib import constants
 from pylib import forwarder
 from pylib.perf import test_runner
 from pylib.utils import test_environment
 
 
+def _GetAllDevices():
+  out_dir = os.path.join(constants.DIR_SOURCE_ROOT, 'out')
+  devices_path = os.path.join(out_dir, '.last_devices')

[jbudorick, 2014/05/29 19:45:25]

Where does the .last_devices file come from?

[bulach, 2014/05/30 09:21:26]

good point! I did some changes in buildbot/bb_device_status_check.py, so there's
now only one place to get this.

+  devices = []
+  try:
+    with open(devices_path) as f:
+      devices = f.read().splitlines()

[jbudorick, 2014/05/29 19:45:25]

I'm concerned about how the tests will behave if the devices change. What if a
device is offline? What if a device falls offline during a run?

[bulach, 2014/05/30 09:21:26]

sorry, my bad! :) I should've given more context here...
up until now, we were optimizing for throughput and reliability: whenever a
device finished its test, it'd pop the next one, and if a device went bad, we'd
just keep going with the remaining ones...

however, this is suboptimal for perf tests: devices have slightly different
characteristics, and the final result was noisy and not as deterministic as we
need.

this change is intentional :) we'll be trading off more determinism, for less
throughput/reliability.

tony can certainly provide far more details here.

+  except IOError as e:
+    logging.error('Unable to find .last_devices [%s]', e)
+    devices = android_commands.GetAttachedDevices()
+  return sorted(devices)
+
+
+def _GetStepsDictFromSingleStep(test_options):
+  # Running a single command, build the tests structure.
+  steps_dict = {
+    'version': 1,
+    'steps': {
+        'single_step': {
+          'device_affinity': 0,
+          'cmd': test_options.single_step
+        },
+    }
+  }
+  return steps_dict
+
+# TODO(bulach): remove once it rolls downstream, crbug.com/378862.
+def _GetStepsDictFromV0(steps_v0):
+  steps_dict = {
+    'version': 1,
+    'steps': {},
+  }
+  affinity = 0
+  for step in steps_v0:
+    steps_dict['steps'][step[0]] = {
+        'device_affinity': affinity,
+        'cmd': step[1],
+    }
+    affinity += 1
+  return steps_dict
+
+
+def _GetStepsDict(test_options):
+  if test_options.single_step:
+    return _GetStepsDictFromSingleStep(test_options)
+  if test_options.steps:
+    with file(test_options.steps, 'r') as f:
+      steps = json.load(f)
+      # TODO(bulach): remove once it rolls downstream, crbug.com/378862.
+      if isinstance(steps, list):
+        return _GetStepsDictFromV0(steps)
+
+      # Already using the new format.
+      assert steps['version'] == 1
+      return steps
+
+
 def Setup(test_options):
   """Create and return the test runner factory and tests.
 
   Args:
     test_options: A PerformanceOptions object.
 
   Returns:
-    A tuple of (TestRunnerFactory, tests).
+    A tuple of (TestRunnerFactory, tests, devices).
   """
   # TODO(bulach): remove this once the bot side lands. BUG=318369
   constants.SetBuildType('Release')
   if os.path.exists(constants.PERF_OUTPUT_DIR):
     shutil.rmtree(constants.PERF_OUTPUT_DIR)
   os.makedirs(constants.PERF_OUTPUT_DIR)
 
   # Before running the tests, kill any leftover server.
   test_environment.CleanupLeftoverProcesses()
   forwarder.Forwarder.UseMultiprocessing()
 
-  if test_options.single_step:
-    # Running a single command, build the tests structure.
-    tests = [['single_step', test_options.single_step]]
+  # We want to keep device affinity, so return all devices ever seen.
+  all_devices = _GetAllDevices()
 
-  if test_options.steps:
-    with file(test_options.steps, 'r') as f:
-      tests = json.load(f)
-
-  # The list is necessary to keep the steps order, but internally
-  # the format is squashed from a list of lists into a single dict:
-  # [["A", "cmd"], ["B", "cmd"]] into {"A": "cmd", "B": "cmd"}
-  sorted_test_names = [i[0] for i in tests]
-  tests_dict = dict(tests)
+  steps_dict = _GetStepsDict(test_options)
+  sorted_step_names = sorted(steps_dict['steps'].keys())
 
   if test_options.test_filter:
-    sorted_test_names = fnmatch.filter(sorted_test_names,
+    sorted_step_names = fnmatch.filter(sorted_step_names,
                                        test_options.test_filter)
-    tests_dict = dict((k, v) for k, v in tests_dict.iteritems()
-                      if k in sorted_test_names)
 
   flaky_steps = []
   if test_options.flaky_steps:
     with file(test_options.flaky_steps, 'r') as f:
       flaky_steps = json.load(f)
 
-  def TestRunnerFactory(device, _shard_index):
+  def TestRunnerFactory(device, shard_index):
     return test_runner.TestRunner(
-        test_options, device, tests_dict, flaky_steps)
+        test_options, device, shard_index, len(all_devices),
+        steps_dict, flaky_steps)
 
-  return (TestRunnerFactory, sorted_test_names)
+  return (TestRunnerFactory, sorted_step_names, all_devices)