Add a recipe to identify culprits for chromium compile failures.

scripts/slave/recipe_modules/chromium_tests/api.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import ast
 import contextlib
 import copy
 import itertools
 import json
 
@@ skipping 280 matching lines @@
           if t.abort_on_failure:
             raise
 
       if failed_tests:
         failed_tests_names = [t.name for t in failed_tests]
         raise self.m.step.StepFailure(
             '%d tests failed: %r' % (len(failed_tests), failed_tests_names))
 
     return test_runner
 
-  def get_compile_targets_and_tests(
+  def get_test_targets_compile_targets_and_tests(
       self, mastername, buildername, master_dict, test_spec,
       override_bot_type=None, override_tests=None):
-    """Returns a tuple: list of compile targets and list of tests.
+    """Returns a tuple: test target list, compile target list, and test list.
 
-    The list of tests includes ones on the triggered testers."""
+    The test list includes ones on the triggered testers."""
 
     bot_config = master_dict.get('builders', {}).get(buildername)
     bot_type = override_bot_type or bot_config.get('bot_type', 'builder_tester')
 
+    if bot_type not in ['builder', 'builder_tester']:
+      return [], [], []
+
     tests = bot_config.get('tests', [])
     if override_tests is not None:
       tests = override_tests
 
-    if bot_type not in ['builder', 'builder_tester']:
-      return [], []
-
-    compile_targets = set(bot_config.get('compile_targets', []))
     tests_including_triggered = list(tests)
     for _, builder_dict in master_dict.get('builders', {}).iteritems():
       if builder_dict.get('parent_buildername') == buildername:
         tests_including_triggered.extend(builder_dict.get('tests', []))
 
+    test_compile_targets = set()
     if bot_config.get('add_tests_as_compile_targets', True):
       for t in tests_including_triggered:
-        compile_targets.update(t.compile_targets(self.m))
+        test_compile_targets.update(t.compile_targets(self.m))
+
+    compile_targets = set(bot_config.get('compile_targets', []))
+    compile_targets.update(test_compile_targets)
 
     # Only add crash_service when we have explicit compile targets.
     if (self.m.platform.is_win and
         compile_targets and
         'all' not in compile_targets):
       compile_targets.add('crash_service')
 
     # Lastly, add any targets the checkout-side test spec told us to use.
     compile_targets.update(test_spec.get(buildername, {}).get(
         'additional_compile_targets', []))
 
-    return sorted(compile_targets), tests_including_triggered
+    return (sorted(test_compile_targets), sorted(compile_targets),

[Paweł Hajdan Jr., 2015/11/24 13:20:41]

Why do we need this change?

[stgao, 2015/11/24 23:19:19]

Was trying to match to new api filter.does_patch_require_compile (build targets
are split into test_targets and additional_compile_targets).

But as Dirk suggested in the recipe below, I reverted this change.

+            tests_including_triggered)
+
+  def get_compile_targets_and_tests(
+      self, mastername, buildername, master_dict, test_spec,
+      override_bot_type=None, override_tests=None):
+    """Returns a tuple: list of compile targets and list of tests.
+
+    The list of tests includes ones on the triggered testers."""
+    _, compile_targets, tests_including_triggered = \
+        self.get_test_targets_compile_targets_and_tests(
+            mastername, buildername, master_dict, test_spec,
+            override_bot_type=override_bot_type, override_tests=override_tests)
+    return compile_targets, tests_including_triggered
 
   def transient_check(self, update_step, command):
     """Runs command, checking for transience if this is a try job.
 
     * command is a function which takes an argument of type (str -> str),
       which is a test name transformation (it adds "with patch" or "without
       patch") and runs the command.
     * update_step is the bot_update step used for deapplying the patch.
     """
     if self.m.tryserver.is_tryserver:
@@ skipping 118 matching lines @@
           mode='dev'
       )
 
   def run_mb_and_compile(self, compile_targets, isolated_targets, name_suffix):
     if self.m.chromium.c.project_generator.tool == 'mb':
       # We don't use the mastername and buildername passed in, because
       # those may be the values of the continuous builder the trybot may
       # be configured to match; we need to use the actual mastername
       # and buildername we're running on, because it may be configured
       # with different MB settings.
-      real_mastername = self.m.properties['mastername']
-      real_buildername = self.m.properties['buildername']
-      self.m.chromium.run_mb(real_mastername, real_buildername,
+      # However, use target_mastername and target_buildername instead if they
+      # are available in the build properties. This gives a recipe running on a
+      # trybot the flexibility to run MB in exactly the same configuration as
+      # the continuous builder it tries to match.
+      target_mastername = (self.m.properties.get('target_mastername') or

[Paweł Hajdan Jr., 2015/11/24 13:20:41]

Let's make this explicit then, i.e. have run_mb_and_compile take mastername and
buildername parameters.

[Dirk Pranke, 2015/11/24 16:35:00]

I actually told stgao to do things this way; because these are passed in as 
build properties to arbitrary recipes, I don't think passing them in as 
parameters actually helps anything, and I'd rather have the logic in one place.

However, reading it again, I think I would add a comment that find it may
need to override the defaults in order to be able to exactly match a given bot.

[stgao, 2015/11/24 23:19:19]

Done.

+                           self.m.properties['mastername'])
+      target_buildername = (self.m.properties.get('target_buildername') or
+                            self.m.properties['buildername'])
+      self.m.chromium.run_mb(target_mastername, target_buildername,
                              isolated_targets=isolated_targets,
                              name='generate_build_files%s' % name_suffix)
 
     self.m.chromium.compile(compile_targets, name='compile%s' % name_suffix)
 
   def tests_for_builder(self, mastername, buildername, update_step, master_dict,
                         override_bot_type=None):
     got_revision = update_step.presentation.properties['got_revision']
 
     bot_config = master_dict.get('builders', {}).get(buildername)
@@ skipping 324 matching lines @@
       result_text = 'MB is enabled for this builder at this revision.'
       log_name = 'Builder MB-ready'
       self.m.step.active_result.presentation.logs[log_name] = [result_text]
       return False
     except (self.m.step.StepFailure, KeyError):
       result_text = 'MB is not enabled for this builder at this revision.'
       log_name = 'Builder NOT MB-ready'
       self.m.step.active_result.presentation.logs[log_name] = [result_text]
       self.m.step.active_result.presentation.status = self.m.step.WARNING
       return True