Android: adds device affinity for perf tests.

build/android/pylib/perf/test_runner.py

Index: build/android/pylib/perf/test_runner.py
diff --git a/build/android/pylib/perf/test_runner.py b/build/android/pylib/perf/test_runner.py
index cedf5734d8315bd6947f12c10b13140246a8ea57..461b1e8477f16fcc2ff263e96b4355ef8edbeb02 100644
--- a/build/android/pylib/perf/test_runner.py
+++ b/build/android/pylib/perf/test_runner.py
@@ -20,8 +20,22 @@ graph data.
 with the step results previously saved. The buildbot will then process the graph
 data accordingly.
 
-
 The JSON steps file contains a dictionary in the format:
+{ "version": int,
+  "steps": {
+    "foo": {
+      "device_affinity": int,

[jbudorick, 2014/05/29 19:45:25]

This might get into how we generate the json, and so it may not be in the scope
of this review.

Why is device_affinity a 'shard_index' and not just the device serial? The
former seems like an obfuscated way of indicating the latter.

If a device disappears from last_devices, we could just clear the affinity for
all tests that have affinity for it.

[bulach, 2014/05/30 09:21:26]

it's complicated :)
as above, we do not want to run tests on devices that nave no affinity.
the index versus serial: it'll make it easier to maintain by just using the
index, due to:
1) ultimately, we don't care about a specific device serial, just that a given
test X always run in the given device in that bot.
2) by using index rather than serial, we'll have less configuration to
maintain.. by using serial, we'd potentially need to hardcode the mapping for
each individual bot, say, Nexus7 Perf, Nexus4 Perf, etc...
the config would end up something like:
"on Nexus7 Perf Bot, run Octane on device serialfoobar".
by using just an index, it's a bit more generic:
"on all bots, run Octane on the 3rd device"

it'll also allow it to be more elastic... for instance, say Nexus7 Perf only
have 2 devices.. by saying "run on the 3rd", we can then "shard % total_devices"
and still keep the affinity.

+      "cmd": "script_to_execute foo"
+    },
+    "bar": {
+      "device_affinity": int,
+      "cmd": "script_to_execute bar"
+    }
+  }
+}

[bulach, 2014/05/29 18:43:32]

tony: this is the information this script needs.
it doesn't matter where it lives, or how it's built: either a static file in
$SRC/, or dynamically  maintained by the buildbot and shared with other
platforms, it doesn't matter much...


downstream, we're keeping checked in under build/sharded_perf_tests.json, so we
can tweak the commands there without messing up with bot configs or anything...

the format itself is fairly flexible, happy to change if you want.
once we agree and this patch lands, then I'll update the json files
downstream...

+
+# TODO(bulach): remove once it rolls downstream, crbug.com/378862.
+The OLD JSON steps file contains a dictionary in the format:
 [
   ["step_name_foo", "script_to_execute foo"],
   ["step_name_bar", "script_to_execute bar"]
@@ -131,17 +145,22 @@ class _HeartBeatLogger(object):
 
 
 class TestRunner(base_test_runner.BaseTestRunner):
-  def __init__(self, test_options, device, tests, flaky_tests):
+  def __init__(self, test_options, device, shard_index, max_shard, tests,
+      flaky_tests):
     """A TestRunner instance runs a perf test on a single device.
 
     Args:
       test_options: A PerfOptions object.
       device: Device to run the tests.
+      shard_index: the index of this device.
+      max_shards: the maximum shard index.
       tests: a dict mapping test_name to command.
       flaky_tests: a list of flaky test_name.
     """
     super(TestRunner, self).__init__(device, None, 'Release')
     self._options = test_options
+    self._shard_index = shard_index
+    self._max_shard = max_shard
     self._tests = tests
     self._flaky_tests = flaky_tests
 
@@ -164,6 +183,16 @@ class TestRunner(base_test_runner.BaseTestRunner):
                              result['name']), 'w') as f:
         f.write(pickle.dumps(result))
 
+  def _CheckDeviceAffinity(self, test_name):
+    """Returns True if test_name has affinity for this shard."""
+    affinity = (self._tests['steps'][test_name]['device_affinity'] %
+                self._max_shard)
+    if self._shard_index == affinity:
+      return True
+    logging.info('Skipping %s on %s (affinity is %s, device is %s)',
+                 test_name, self.device_serial, affinity, self._shard_index)
+    return False
+
   def _LaunchPerfTest(self, test_name):
     """Runs a perf test.
 
@@ -173,6 +202,9 @@ class TestRunner(base_test_runner.BaseTestRunner):
     Returns:
       A tuple containing (Output, base_test_result.ResultType)
     """
+    if not self._CheckDeviceAffinity(test_name):
+      return '', base_test_result.ResultType.PASS
+
     try:
       logging.warning('Unmapping device ports')
       forwarder.Forwarder.UnmapAllDevicePorts(self.device)
@@ -181,7 +213,8 @@ class TestRunner(base_test_runner.BaseTestRunner):
       logging.error('Exception when tearing down device %s', e)
 
     cmd = ('%s --device %s' %
-           (self._tests[test_name], self.device.old_interface.GetDevice()))
+           (self._tests['steps'][test_name]['cmd'],
+            self.device_serial))
     logging.info('%s : %s', test_name, cmd)
     start_time = datetime.datetime.now()
 
@@ -212,7 +245,7 @@ class TestRunner(base_test_runner.BaseTestRunner):
       exit_code = -1
     logging.info('%s : exit_code=%d in %d secs at %s',
                  test_name, exit_code, (end_time - start_time).seconds,
-                 self.device.old_interface.GetDevice())
+                 self.device_serial)
     result_type = base_test_result.ResultType.FAIL
     if exit_code == 0:
       result_type = base_test_result.ResultType.PASS
@@ -231,7 +264,7 @@ class TestRunner(base_test_runner.BaseTestRunner):
         'actual_exit_code': actual_exit_code,
         'result_type': result_type,
         'total_time': (end_time - start_time).seconds,
-        'device': self.device.old_interface.GetDevice(),
+        'device': self.device_serial,
         'cmd': cmd,
     }
     self._SaveResult(persisted_result)