[Findit] Updating extractors.py to account for new compile failure stdio format

appengine/findit/waterfall/extractors.py

Index: appengine/findit/waterfall/extractors.py
diff --git a/appengine/findit/waterfall/extractors.py b/appengine/findit/waterfall/extractors.py
index a9645bfcd5e92d2ca324324a175f4007c956d138..0900c615223ef41a4685844996c5659c2dff7db7 100644
--- a/appengine/findit/waterfall/extractors.py
+++ b/appengine/findit/waterfall/extractors.py
@@ -125,7 +125,10 @@ class CompileStepExtractor(Extractor):
 
   NINJA_FAILURE_END_LINE_PREFIX = 'ninja: build stopped'
   NINJA_ERROR_LINE_PREFIX = 'ninja: error'
-  ERROR_LINE_END_PATTERN = re.compile(r'^\d+ errors? generated.')
+  ERROR_LINE_END_PATTERN1 = re.compile(r'^\d+ errors? generated.')
+  ERROR_LINE_END_PATTERN2 = re.compile(r'failed with exit code \d+')

[chanli, 2016/04/29 18:26:34]

Question: in the stdio file of the
example(https://build.chromium.org/p/chromium.webkit/builders/WebKit%20Linux/builds/63875)
that was mentioned in bug: I couldn't find 'failed with exit code' but 'step
returned non-zero exit code: 1'. Is this what expected or I just looked into
wrong place?

[lijeffrey, 2016/04/29 19:23:35]

The example for "failed with exit code" was from
https://findit-for-me.appspot.com/waterfall/failure-log?url=https://build.chr...

"returned non-zero exit code: 1" is outside wher we want to be looking. We
should only be looking between

FAILED: (target that failed) <-- this is part of the new format
command that failed <-- extract targets from the first line
... <-- extract files
"n error(s) generated" or "linker command failed with exit code 1" <-- stop here

If neither of those lines is encountered while scanning a FAILED block but we
encounter something like [123/456], then we also stop.

+  OUTSIDE_FAILURE_LINE_PATTERN = re.compile(r'\[\d+/\d+\]')
+
   IOS_ERROR_LINE_START_PREFIX = 'CompileC'
 
   IOS_BUILDER_NAMES_FOR_COMPILE = ['iOS_Simulator_(dbg)', 'iOS_Device']
@@ -133,15 +136,14 @@ class CompileStepExtractor(Extractor):
 
   GOMA_COMPILER_PREFIX = (
       r'\S*/gomacc '
-       '\S+(?:clang\+\+|androideabi-gcc|androideabi-g\+\+)'
+      '\S+(?:clang\+\+|androideabi-gcc|androideabi-g\+\+)'
   )
   STRICT_COMPILE_FAILURE_PATTEN = re.compile(
-      r'^FAILED: %s -MMD -MF (obj/[^\s-]+\.o)\.d .* '
-       '-c ([^\s-]+(?:cc|c|cpp|m|mm)) -o \\1$' % (
-           GOMA_COMPILER_PREFIX))
+      r'^%s -MMD -MF (obj/[^\s-]+\.o)\.d .* '
+      '-c ([^\s-]+(?:cc|c|cpp|m|mm)) -o \\1$' % GOMA_COMPILER_PREFIX)
 
   STRICT_LINK_FAILURE_PATTEN = re.compile(
-      r'^FAILED: %s -Wl,.* -o (\S+) -Wl,--start-group .*$' % (
+      r'^%s -Wl,.* -o (\S+) -Wl,--start-group .*$' % (
           GOMA_COMPILER_PREFIX))
 
   def ExtractFailedOutputNodes(self, line, signal):
@@ -214,13 +216,14 @@ class CompileStepExtractor(Extractor):
   def Extract(self, failure_log, test_name, step_name, bot_name, master_name):
     signal = FailureSignal()
     failure_started = False
+    is_build_command_line = False
 
     if (master_name == self.MAC_MASTER_NAME_FOR_COMPILE and
         bot_name in self.IOS_BUILDER_NAMES_FOR_COMPILE):
       error_lines = []
       for line in reversed(failure_log.splitlines()):
         if (not failure_started and
-            self.ERROR_LINE_END_PATTERN.match(line)):
+            self.ERROR_LINE_END_PATTERN1.match(line)):
           failure_started = True
           continue
 
@@ -237,24 +240,32 @@ class CompileStepExtractor(Extractor):
           master_name, bot_name)
       for line in failure_log.splitlines():
         if line.startswith(self.FAILURE_START_LINE_PREFIX):
+          is_build_command_line = True
+          if not failure_started:
+            failure_started = True
+          continue  # pragma: no cover

[stgao, 2016/04/29 16:38:24]

Hm, I think this should be executed in a common testcase. Is there anything
wrong?

[lijeffrey, 2016/04/29 19:23:35]

It seems there's a bug in coverage

a = False 
for i in range(1, 4): 
  if i == 2: 
    if not a:
      a = True 
    # b = 3 
    continue 
  a = False

as it is, with code coverage the "continue" line will be marked as not executed
but with "# b = 3" uncommented, the continue line is executed.

To work around this I'll put is_build_command_line = True after the if statement
instead of before

+        elif is_build_command_line:
           if strict_regex:
             self.ExtractFailedOutputNodes(line, signal)
           else:
             self.GetFailedTarget(line, signal)
-          if not failure_started:
-            failure_started = True
-          continue  # pragma: no cover
+          is_build_command_line = False
+          continue
         elif self.FAILURE_WITH_ERROR_PATTERN.match(line):
           # It is possible the target and source file associated with a compile
           # failure is logged outside a 'FAILED: ... 1 error generated' block,
-          # so extract regardless of failure_started.
+          # so extract targets regardless of failure_started.
           if not strict_regex:
             self.GetFailedTarget(line, signal)
-        elif failure_started and self.ERROR_LINE_END_PATTERN.match(line):
+        elif (failure_started and
+              (self.ERROR_LINE_END_PATTERN1.match(line) or
+               self.ERROR_LINE_END_PATTERN2.search(line) or
+               self.OUTSIDE_FAILURE_LINE_PATTERN.match(line))):
           failure_started = False
         elif failure_started and line.startswith(
             self.NINJA_FAILURE_END_LINE_PREFIX):  # pragma: no cover
           break
+
         if failure_started or line.startswith(self.NINJA_ERROR_LINE_PREFIX):
           # either within the compile errors or is a ninja error.
           self.ExtractFiles(line, signal)