[Findit] Extract failed output nodes from stdio log of compile step.

appengine/findit/waterfall/extractors.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import os
 import re
 
 from waterfall import extractor_util
 from waterfall import waterfall_config
@@ skipping 199 matching lines @@
         match = self.WINDOWS_LINK_FAILURE_PATTERN.search(line)
         if match:
           # Failure is a linker error.
           target = match.group(1)
           signal.AddTarget({'target': target})
 
   def Extract(self, failure_log, test_name, step_name, bot_name, master_name):
     signal = FailureSignal()
     failure_started = False
     is_build_command_line = False
+    failed_output_nodes = set()
 
     if (master_name == self.MAC_MASTER_NAME_FOR_COMPILE and
         bot_name in self.IOS_BUILDER_NAMES_FOR_COMPILE):
       error_lines = []
       for line in reversed(failure_log.splitlines()):
         if (not failure_started and
             self.ERROR_LINE_END_PATTERN1.match(line)):
           failure_started = True
           continue
 
         if failure_started:
           if line.startswith(self.IOS_ERROR_LINE_START_PREFIX):
             failure_started = False
             for l in error_lines[:-4]:
               self.ExtractFiles(l, signal)
             error_lines = []
           else:
             error_lines.append(line)
     else:
       strict_regex = waterfall_config.EnableStrictRegexForCompileLinkFailures(
           master_name, bot_name)
       for line in failure_log.splitlines():
         if line.startswith(self.FAILURE_START_LINE_PREFIX):
           if not failure_started:
             failure_started = True
+          line = line[len(self.FAILURE_START_LINE_PREFIX):]
+          while line:

[chanli, 2016/04/30 02:24:58]

Is it possible that the line is like:
FAILED: a.o "something in quotes" b.o "some other thing in quotes"?

[stgao, 2016/05/01 05:12:28]

In theory, this is possible.
But in practice, when with quote, it would be more like:
FAILED: "some target with space".

And this is handled by the code below.

+            quote_index = line.find('"')

[lijeffrey, 2016/04/30 03:44:25]

nit: it seems since this is a temporary solution, maybe extract this into a
separate function that takes line and returns a set that will be
failed_output_nodes?

i.e. 

def _GetOutputNodesFromLine(line):
  failed_output_nodes = set()
  while line:
    quote_index = line.find('"')
    ...
  return failed_output_nodes

then later

line = line[len(self.FAILURE_START_LINE_PREFIX):]
failed_ouptut_nodes = _GetOutputNodesFromLine(line)
is_build_command_line = True
...


This way the function can be more easily tested and removed when this temporary
solution is no longer needed

[stgao, 2016/05/01 05:12:28]

This is a good idea. 
Done, but with additional change.

+            if quote_index < 0:
+              sub_part = line
+              remaining_part = None
+            else:
+              sub_part = line[:quote_index]
+              match_quote_index = line.find('"', quote_index + 1)
+              if match_quote_index < 0:  # Unexpected format.
+                failed_output_nodes.clear()
+                break
+              failed_output_nodes.add(
+                  line[quote_index + 1: match_quote_index])
+              remaining_part = line[match_quote_index + 1:]
+            line = remaining_part
+
+            for node in sub_part.split(' '):
+              node = node.strip()
+              if node:
+                failed_output_nodes.add(node)
+
           is_build_command_line = True
           continue
         elif is_build_command_line:
           if strict_regex:
             self.ExtractFailedOutputNodes(line, signal)
           else:
             self.GetFailedTarget(line, signal)
           is_build_command_line = False
           continue
         elif self.FAILURE_WITH_ERROR_PATTERN.match(line):
           # It is possible the target and source file associated with a compile
           # failure is logged outside a 'FAILED: ... 1 error generated' block,
           # so extract targets regardless of failure_started.
           if not strict_regex:
             self.GetFailedTarget(line, signal)
         elif (failure_started and
               (self.ERROR_LINE_END_PATTERN1.match(line) or
                self.ERROR_LINE_END_PATTERN2.search(line) or
                self.OUTSIDE_FAILURE_LINE_PATTERN.match(line))):
           failure_started = False
         elif failure_started and line.startswith(
             self.NINJA_FAILURE_END_LINE_PREFIX):  # pragma: no cover
           break
 
         if failure_started or line.startswith(self.NINJA_ERROR_LINE_PREFIX):
           # either within the compile errors or is a ninja error.
           self.ExtractFiles(line, signal)
 
+    signal.failed_output_nodes = sorted(failed_output_nodes)
     return signal
 
 
 class CheckPermExtractor(Extractor):
   """For CheckPerm, only extracts files."""
 
   def Extract(self, failure_log, *_):
     signal = FailureSignal()
 
     for line in reversed(failure_log.splitlines()):
@@ skipping 242 matching lines @@
 def ExtractSignal(master_name, bot_name, step_name, test_name, failure_log):
   """Uses an appropriate extractor to extract failure signals.
 
   Returns:
     A FailureSignal.
   """
   # Fall back to a general-but-maybe-not-accurate extractor.
   extractor_class = EXTRACTORS.get(step_name, GeneralExtractor)
   return extractor_class().Extract(
       failure_log, test_name, step_name, bot_name, master_name)