Support sharding for swarmed isolated script tests.

scripts/slave/recipe_modules/swarming/api.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import functools
 
 from recipe_engine.types import freeze
 from recipe_engine import recipe_api
 
@@ skipping 685 matching lines @@
 
         # Show any remaining isolated outputs (such as logcats).
         # Note that collect_gtest_task.py uses the default summary.json, which
         # only has 'outputs_ref' instead of the deprecated 'isolated_out'.
         for index, shard in enumerate(swarming_summary.get('shards', [])):
           outputs_ref = shard.get('outputs_ref')
           if outputs_ref:
             link_name = 'shard #%d isolated out' % index
             p.links[link_name] = outputs_ref['view_url']
 
+
+  def _merge_isolated_script_shards(self, task, step_result):
+    # This code is unfortunately specialized to the "simplified"
+    # JSON format that used to be the standard for recipes. The
+    # isolated scripts should be changed to use the now-standard
+    # Chromium JSON test results format:
+    # https://www.chromium.org/developers/the-json-test-results-format
+    # . Note that gtests, above, don't seem to conform to this
+    # format yet, so it didn't seem like a good prerequisite to
+    # switch the isolated tests over when adding sharding support.
+    #
+    # These are the only keys we pay attention to in the output JSON.
+    merged_results = {
+      'successes': [],
+      'failures': [],
+      'valid': True,
+    }
+    for i in xrange(task.shards):
+      results_raw = step_result.raw_io.output_dir[

[Vadim Sh., 2016/06/17 00:31:11]

I think at the very minimum we can acknowledge in code that sometimes
output.json may be missing by writing:

path = self.m.path.join(str(i), 'output.json')
if path not in step_result.raw_io.output_dir:
  raise Exception("no results from shard #%d" % i)

(raw_io.output_dir seems to be a regular dict, so "in" checks presences of the
file)

+          self.m.path.join(str(i), 'output.json')]
+      results_json = self.m.json.loads(results_raw)
+      for key in merged_results:
+        if key in results_json:
+          if isinstance(merged_results[key], list):
+            merged_results[key].extend(results_json[key])
+          elif isinstance(merged_results[key], bool):
+            merged_results[key] = merged_results[key] and results_json[key]
+          else:
+            raise recipe_api.InfraFailure(
+              'Unknown key type ' + type(merged_results[key]) +
+              ' when handling key ' + key + '.')  # pragma: no cover
+    return merged_results
+
   def _isolated_script_collect_step(self, task, **kwargs):
     step_test_data = kwargs.pop('step_test_data', None)
     if not step_test_data:
       step_test_data = self.m.test_utils.test_api.canned_isolated_script_output(
           passing=True, is_win=self.m.platform.is_win, swarming=True)
 
     args=self.get_collect_cmd_args(task)
     args.extend(['--task-output-dir', self.m.raw_io.output_dir()])
 
     try:
@@ skipping 13 matching lines @@
         step_result = self.m.step.active_result
         outdir_json = self.m.json.dumps(step_result.raw_io.output_dir, indent=2)
         step_result.presentation.logs['outdir_json'] = outdir_json.splitlines()
 
         # Check if it's an internal failure.
         summary = self.m.json.loads(
             step_result.raw_io.output_dir['summary.json'])
         if any(shard['internal_failure'] for shard in summary['shards']):
           raise recipe_api.InfraFailure('Internal swarming failure.')
 
-        # TODO(nednguyen, kbr): Combine isolated script results from multiple
-        # shards rather than assuming that there is always just one shard.
-        assert len(summary['shards']) == 1
-        results_raw = step_result.raw_io.output_dir[
-            self.m.path.join('0', 'output.json')]
-        step_result.isolated_script_results = self.m.json.loads(results_raw)
+        # Always show the shards' links in the collect step. (It looks
+        # like show_isolated_out_in_collect_step is false by default
+        # in recipe runs.)
+        links = step_result.presentation.links
+        for index in xrange(task.shards):
+          url = task.get_shard_view_url(index)
+          if url:
+            links['shard #%d' % index] = url
+
+        step_result.isolated_script_results = \
+          self._merge_isolated_script_shards(task, step_result)
 
         self._display_pending(summary, step_result.presentation)
       except Exception as e:
         self.m.step.active_result.presentation.logs['no_results_exc'] = [str(e)]
         self.m.step.active_result.isolated_script_results = None
 
   def _get_step_name(self, prefix, task):
     """SwarmingTask -> name of a step of a waterfall.
 
     Will take a task title (+ step name prefix) and append OS dimension to it.
@@ skipping 173 matching lines @@
 
   def get_shard_view_url(self, index):
     """Returns URL of HTML page with shard details or None if not available.
 
     Works only after the task has been successfully triggered.
     """
     if self._trigger_output and self._trigger_output.get('tasks'):
       for shard_dict in self._trigger_output['tasks'].itervalues():
         if shard_dict['shard_index'] == index:
           return shard_dict['view_url']