[Findit] Collect failed output nodes of compile and link failures by strict regex

appengine/findit/waterfall/extractors.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import logging
 import os
 import re
 
 from waterfall import extractor_util
+from waterfall import waterfall_config
 from waterfall.extractor import Extractor
 from waterfall.failure_signal import FailureSignal
 
 
 class GeneralExtractor(Extractor):
   """A general extractor.
 
   It extracts file name and line numbers.
   """
   INDIRECT_LEAK_MARKER_PATTERN = re.compile(
@@ skipping 103 matching lines @@
   WINDOWS_LINK_FAILURE_PATTERN = re.compile(r'/OUT:([^\s-]+)')
 
   NINJA_FAILURE_END_LINE_PREFIX = 'ninja: build stopped'
   NINJA_ERROR_LINE_PREFIX = 'ninja: error'
   ERROR_LINE_END_PATTERN = re.compile(r'^\d+ errors? generated.')
   IOS_ERROR_LINE_START_PREFIX = 'CompileC'
 
   IOS_BUILDER_NAMES_FOR_COMPILE = ['iOS_Simulator_(dbg)', 'iOS_Device']
   MAC_MASTER_NAME_FOR_COMPILE = 'chromium.mac'
 
+  GOMA_COMPILER_PREFIX = (
+      r'\S*/gomacc '
+       '\S+(?:clang\+\+|androideabi-gcc|androideabi-g\+\+)'
+  )
+  STRICT_COMPILE_FAILURE_PATTEN = re.compile(
+      r'^FAILED: %s -MMD -MF (obj/[^\s-]+\.o)\.d .* '
+       '-c ([^\s-]+(?:cc|c|cpp|m|mm)) -o \\1$' % (
+           GOMA_COMPILER_PREFIX))
+
+  STRICT_LINK_FAILURE_PATTEN = re.compile(
+      r'^FAILED: %s -Wl,.* -o (\S+) -Wl,--start-group .*$' % (
+          GOMA_COMPILER_PREFIX))
+
+  def ExtractFailedOutputNodes(self, line, signal):
+    match = self.STRICT_COMPILE_FAILURE_PATTEN.match(line)
+    if match:
+      target = match.group(1)
+      source = match.group(2)
+      signal.AddTarget({
+          'target': target,
+          'source': source,
+      })
+    else:
+      match = self.STRICT_LINK_FAILURE_PATTEN.match(line)
+      if match:
+        target = match.group(1)
+        signal.AddTarget({
+            'target': target,
+        })
+      else:
+        logging.warn('Unknown failure type: %s', line)
+
   def GetFailedTarget(self, line, signal):
     match = self.LINUX_FAILED_SOURCE_TARGET_PATTERN.search(line)
 
     if match:
       # For non-windows builds, build lines should contain either gomacc or
       # clang++, so only search for source/target files after the call to
       # gomacc/clang++ to avoid extracting false positives in the failure line.
       build_line_match = self.LINUX_BUILD_COMMAND_PATTERN.search(line)
       if not build_line_match:
         return
@@ skipping 50 matching lines @@
 
         if failure_started:
           if line.startswith(self.IOS_ERROR_LINE_START_PREFIX):
             failure_started = False
             for l in error_lines[:-4]:
               self.ExtractFiles(l, signal)
             error_lines = []
           else:
             error_lines.append(line)
     else:
+      strict_regex = waterfall_config.EnableStrictRegexForCompileLinkFailures(
+          master_name, bot_name)
       for line in failure_log.splitlines():
         if line.startswith(self.FAILURE_START_LINE_PREFIX):
-          self.GetFailedTarget(line, signal)
+          if strict_regex:
+            self.ExtractFailedOutputNodes(line, signal)
+          else:
+            self.GetFailedTarget(line, signal)
           if not failure_started:
             failure_started = True
           continue  # pragma: no cover
         elif self.FAILURE_WITH_ERROR_PATTERN.match(line):
           # It is possible the target and source file associated with a compile
           # failure is logged outside a 'FAILED: ... 1 error generated' block,
           # so extract regardless of failure_started.
-          self.GetFailedTarget(line, signal)
+          if not strict_regex:
+            self.GetFailedTarget(line, signal)
         elif failure_started and self.ERROR_LINE_END_PATTERN.match(line):
           failure_started = False
         elif failure_started and line.startswith(
             self.NINJA_FAILURE_END_LINE_PREFIX):  # pragma: no cover
           break
         if failure_started or line.startswith(self.NINJA_ERROR_LINE_PREFIX):
           # either within the compile errors or is a ninja error.
           self.ExtractFiles(line, signal)
 
     return signal
@@ skipping 251 matching lines @@
 def ExtractSignal(master_name, bot_name, step_name, test_name, failure_log):
   """Uses an appropriate extractor to extract failure signals.
 
   Returns:
     A FailureSignal.
   """
   # Fall back to a general-but-maybe-not-accurate extractor.
   extractor_class = EXTRACTORS.get(step_name, GeneralExtractor)
   return extractor_class().Extract(
       failure_log, test_name, step_name, bot_name, master_name)