Android: adds device affinity for perf tests.

build/android/pylib/perf/test_runner.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs perf tests.
 
 Our buildbot infrastructure requires each slave to run steps serially.
 This is sub-optimal for android, where these steps can run independently on
 multiple connected devices.
 
 The buildbots will run this script multiple times per cycle:
 - First: all steps listed in --steps in will be executed in parallel using all
 connected devices. Step results will be pickled to disk. Each step has a unique
 name. The result code will be ignored if the step name is listed in
 --flaky-steps.
 The buildbot will treat this step as a regular step, and will not process any
 graph data.
 
 - Then, with -print-step STEP_NAME: at this stage, we'll simply print the file
 with the step results previously saved. The buildbot will then process the graph
 data accordingly.
 
+The JSON steps file contains a dictionary in the format:
+{ "version": int,
+  "steps": {
+    "foo": {
+      "device_affinity": int,

[jbudorick, 2014/05/29 19:45:25]

This might get into how we generate the json, and so it may not be in the scope
of this review.

Why is device_affinity a 'shard_index' and not just the device serial? The
former seems like an obfuscated way of indicating the latter.

If a device disappears from last_devices, we could just clear the affinity for
all tests that have affinity for it.

[bulach, 2014/05/30 09:21:26]

it's complicated :)
as above, we do not want to run tests on devices that nave no affinity.
the index versus serial: it'll make it easier to maintain by just using the
index, due to:
1) ultimately, we don't care about a specific device serial, just that a given
test X always run in the given device in that bot.
2) by using index rather than serial, we'll have less configuration to
maintain.. by using serial, we'd potentially need to hardcode the mapping for
each individual bot, say, Nexus7 Perf, Nexus4 Perf, etc...
the config would end up something like:
"on Nexus7 Perf Bot, run Octane on device serialfoobar".
by using just an index, it's a bit more generic:
"on all bots, run Octane on the 3rd device"

it'll also allow it to be more elastic... for instance, say Nexus7 Perf only
have 2 devices.. by saying "run on the 3rd", we can then "shard % total_devices"
and still keep the affinity.

+      "cmd": "script_to_execute foo"
+    },
+    "bar": {
+      "device_affinity": int,
+      "cmd": "script_to_execute bar"
+    }
+  }
+}

[bulach, 2014/05/29 18:43:32]

tony: this is the information this script needs.
it doesn't matter where it lives, or how it's built: either a static file in
$SRC/, or dynamically  maintained by the buildbot and shared with other
platforms, it doesn't matter much...


downstream, we're keeping checked in under build/sharded_perf_tests.json, so we
can tweak the commands there without messing up with bot configs or anything...

the format itself is fairly flexible, happy to change if you want.
once we agree and this patch lands, then I'll update the json files
downstream...

 
-The JSON steps file contains a dictionary in the format:
+# TODO(bulach): remove once it rolls downstream, crbug.com/378862.
+The OLD JSON steps file contains a dictionary in the format:
 [
   ["step_name_foo", "script_to_execute foo"],
   ["step_name_bar", "script_to_execute bar"]
 ]
 
 This preserves the order in which the steps are executed.
 
 The JSON flaky steps file contains a list with step names which results should
 be ignored:
 [
@@ skipping 89 matching lines @@
     if now - self._tick >= _HeartBeatLogger._PRINT_INTERVAL:
       self._tick = now
       print '--single-step output length %d' % self._len
       sys.stdout.flush()
 
   def stop(self):
     self._stopped.set()
 
 
 class TestRunner(base_test_runner.BaseTestRunner):
-  def __init__(self, test_options, device, tests, flaky_tests):
+  def __init__(self, test_options, device, shard_index, max_shard, tests,
+      flaky_tests):
     """A TestRunner instance runs a perf test on a single device.
 
     Args:
       test_options: A PerfOptions object.
       device: Device to run the tests.
+      shard_index: the index of this device.
+      max_shards: the maximum shard index.
       tests: a dict mapping test_name to command.
       flaky_tests: a list of flaky test_name.
     """
     super(TestRunner, self).__init__(device, None, 'Release')
     self._options = test_options
+    self._shard_index = shard_index
+    self._max_shard = max_shard
     self._tests = tests
     self._flaky_tests = flaky_tests
 
   @staticmethod
   def _IsBetter(result):
     if result['actual_exit_code'] == 0:
       return True
     pickled = os.path.join(constants.PERF_OUTPUT_DIR,
                            result['name'])
     if not os.path.exists(pickled):
       return True
     with file(pickled, 'r') as f:
       previous = pickle.loads(f.read())
     return result['actual_exit_code'] < previous['actual_exit_code']
 
   @staticmethod
   def _SaveResult(result):
     if TestRunner._IsBetter(result):
       with file(os.path.join(constants.PERF_OUTPUT_DIR,
                              result['name']), 'w') as f:
         f.write(pickle.dumps(result))
 
+  def _CheckDeviceAffinity(self, test_name):
+    """Returns True if test_name has affinity for this shard."""
+    affinity = (self._tests['steps'][test_name]['device_affinity'] %
+                self._max_shard)
+    if self._shard_index == affinity:
+      return True
+    logging.info('Skipping %s on %s (affinity is %s, device is %s)',
+                 test_name, self.device_serial, affinity, self._shard_index)
+    return False
+
   def _LaunchPerfTest(self, test_name):
     """Runs a perf test.
 
     Args:
       test_name: the name of the test to be executed.
 
     Returns:
       A tuple containing (Output, base_test_result.ResultType)
     """
+    if not self._CheckDeviceAffinity(test_name):
+      return '', base_test_result.ResultType.PASS
+
     try:
       logging.warning('Unmapping device ports')
       forwarder.Forwarder.UnmapAllDevicePorts(self.device)
       self.device.old_interface.RestartAdbdOnDevice()
     except Exception as e:
       logging.error('Exception when tearing down device %s', e)
 
     cmd = ('%s --device %s' %
-           (self._tests[test_name], self.device.old_interface.GetDevice()))
+           (self._tests['steps'][test_name]['cmd'],
+            self.device_serial))
     logging.info('%s : %s', test_name, cmd)
     start_time = datetime.datetime.now()
 
     timeout = 5400
     if self._options.no_timeout:
       timeout = None
     full_cmd = cmd
     if self._options.dry_run:
       full_cmd = 'echo %s' % cmd
 
@@ skipping 10 matching lines @@
         withexitstatus=True, logfile=logfile, timeout=timeout,
         env=os.environ)
     if self._options.single_step:
       # Stop the logger.
       logfile.stop()
     end_time = datetime.datetime.now()
     if exit_code is None:
       exit_code = -1
     logging.info('%s : exit_code=%d in %d secs at %s',
                  test_name, exit_code, (end_time - start_time).seconds,
-                 self.device.old_interface.GetDevice())
+                 self.device_serial)
     result_type = base_test_result.ResultType.FAIL
     if exit_code == 0:
       result_type = base_test_result.ResultType.PASS
     actual_exit_code = exit_code
     if test_name in self._flaky_tests:
       # The exit_code is used at the second stage when printing the
       # test output. If the test is flaky, force to "0" to get that step green
       # whilst still gathering data to the perf dashboards.
       # The result_type is used by the test_dispatcher to retry the test.
       exit_code = 0
 
     persisted_result = {
         'name': test_name,
         'output': output,
         'exit_code': exit_code,
         'actual_exit_code': actual_exit_code,
         'result_type': result_type,
         'total_time': (end_time - start_time).seconds,
-        'device': self.device.old_interface.GetDevice(),
+        'device': self.device_serial,
         'cmd': cmd,
     }
     self._SaveResult(persisted_result)
 
     return (output, result_type)
 
   def RunTest(self, test_name):
     """Run a perf test on the device.
 
     Args:
       test_name: String to use for logging the test result.
 
     Returns:
       A tuple of (TestRunResults, retry).
     """
     _, result_type = self._LaunchPerfTest(test_name)
     results = base_test_result.TestRunResults()
     results.AddResult(base_test_result.BaseTestResult(test_name, result_type))
     retry = None
     if not results.DidRunPass():
       retry = test_name
     return results, retry