Collect failed "target"s from ninja in the compile step on Waterfall.

scripts/slave/compile.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A tool to build chrome, executed by buildbot.
 
   When this is run, the current directory (cwd) should be the outer build
   directory (e.g., chrome-release/build/).
 
   For a list of command-line options, call this script with '--help'.
 """
 
 import datetime
 import errno
+import json
 import multiprocessing
 import optparse
 import os
 import re
 import shlex
 import sys
 import time
 
 from common import chromium_utils
 from slave import build_directory
@@ skipping 42 matching lines @@
       fh = sys.stdout
     fh.write('Environment variables modified in compile.py:\n')
     for k in sorted(list(self.overrides)):
       if k in self:
         fh.write('  %s=%s\n' % (k, self[k]))
       else:
         fh.write('  %s (removed)\n' % k)
     fh.write('\n')
 
 
+class NinjaFailedTargetCollector(object):
+  """This is to scrape the ninja output to collect failed "target"s.
+
+  A target here is actually an output node in the ninja build graph. Thus it
+  could be an object file, a lib, an executable, output of an action, etc.
+
+  When a build edge fails, its output nodes will be printed in a single line
+  in the following format these without quotes:
+      "FAILED: obj/path/to/file1.o obj/path/to/file2.o "
+      "FAILED: gen/path/to/name.cc gen/path/to/name.h "
+
+  If a target is specified in the test spec but it is not defined in the gn or
+  gyp build config, an "unknown target" error occurs in these two possible
+  formats without quotes:
+      "ninja: error: unknown target 'executable_target'"
+      "ninja: error: unknown target 'exe1', did you mean 'exe2'?"
+  """
+  NINJA_FAILURE_PREFIX = 'FAILED: '
+  NINJA_UNKNOWN_TARGET_PREFIX = 'ninja: error: unknown target '
+
+  def __init__(self, file_path):
+    self.file_path = file_path
+    self.failed_targets = []
+    self.unknown_targets = []
+    self.unrecognized_format = False
+
+  def extract(self, line):
+    if line.startswith(self.NINJA_FAILURE_PREFIX):
+      for node in line[len(self.NINJA_FAILURE_PREFIX):].split(' '):
+        node = node.strip()
+        if node:
+          self.failed_targets.append(node)
+    elif line.startswith(self.NINJA_UNKNOWN_TARGET_PREFIX):
+      parts = line[len(self.NINJA_UNKNOWN_TARGET_PREFIX):].split('\'')
+      if len(parts) >= 3 and parts[1].strip():
+        self.unknown_targets.append(parts[1].strip())
+      else:
+        self.unrecognized_format = True
+
+  def dump_as_json(self):
+    with open(self.file_path, 'wb') as f:
+      json.dump({
+          'failed_targets': self.failed_targets,
+          'unknown_targets': self.unknown_targets,
+          'unrecognized_format': self.unrecognized_format,
+      }, f)
+
+
 def ReadHKLMValue(path, value):
   """Retrieve the install path from the registry for Visual Studio 8.0 and
   Incredibuild."""
   # Only available on Windows.
   # pylint: disable=F0401
   import win32api, win32con
   try:
     regkey = win32api.RegOpenKeyEx(win32con.HKEY_LOCAL_MACHINE, path, 0,
                                    win32con.KEY_READ)
     value = win32api.RegQueryValueEx(regkey, value)[0]
@@ skipping 858 matching lines @@
             ['build%d-m4' % x for x in xrange(45, 48)]):
           return min(10 * number_of_processors, 200)
 
         return 50
 
       goma_jobs = determine_goma_jobs()
       command.append('-j%d' % goma_jobs)
 
     # Run the build.
     env.print_overrides()
-    exit_status = chromium_utils.RunCommand(command, env=env)
+    kwargs = {
+        'env': env,
+    }
+    failed_target_collector = None
+    if options.ninja_compile_failure:
+      failed_target_collector = NinjaFailedTargetCollector(
+          options.ninja_compile_failure)
+      kwargs['parser_func'] = failed_target_collector.extract
+    exit_status = chromium_utils.RunCommand(command, **kwargs)
     if exit_status == 0 and options.ninja_ensure_up_to_date:
       # Run the build again if we want to check that the no-op build is clean.
       filter_obj = EnsureUpToDateFilter()
       # Append `-d explain` to help diagnose in the failure case.
       command += ['-d', 'explain']
       chromium_utils.RunCommand(command, env=env, filter_obj=filter_obj)
       if not filter_obj.was_up_to_date:
         print 'Failing build because ninja reported work to do.'
         print 'This means that after completing a compile, another was run and'
         print 'it resulted in still having work to do (that is, a no-op build'
         print 'wasn\'t a no-op). Consult the first "ninja explain:" line for a'
         print 'likely culprit.'
         return 1
+    if exit_status != 0 and failed_target_collector:
+        failed_target_collector.dump_as_json()
     return exit_status
   finally:
     goma_teardown(options, env, exit_status)
 
     override_gsutil = None
     if options.gsutil_py_path:
       override_gsutil = [sys.executable, options.gsutil_py_path]
 
     goma_utils.UploadNinjaLog(
         options.target_output_dir, options.compiler, command, exit_status,
@@ skipping 293 matching lines @@
                                 ' credentials')
   option_parser.add_option('--verbose', action='store_true')
   option_parser.add_option('--gsutil-py-path',
                            help='Specify path to gsutil.py script.')
   option_parser.add_option('--ninja-path', default='ninja',
                            help='Specify path to the ninja tool.')
   option_parser.add_option('--ninja-ensure-up-to-date', action='store_true',
                            help='Checks the output of the ninja builder to '
                                 'confirm that a second compile immediately '
                                 'the first is a no-op.')
+  option_parser.add_option('--ninja-compile-failure',
+                           help='Specify a file to dump detailed info of '
+                                'compile failure from ninja. Effective only '
+                                'when building with ninja.')
 
   options, args = option_parser.parse_args()
 
   if not options.src_dir:
     options.src_dir = 'src'
   options.src_dir = os.path.abspath(options.src_dir)
 
   options.build_dir = os.path.abspath(build_directory.GetBuildOutputDirectory(
         os.path.basename(options.src_dir)))
 
@@ skipping 58 matching lines @@
       sys.stderr.write('Unknown build tool %s.\n' % repr(options.build_tool))
       return 2
 
   options.target_output_dir = get_target_build_dir(args, options)
 
   return main(options, args)
 
 
 if '__main__' == __name__:
   sys.exit(real_main())