[Findit] Add a sub-pipeline to analyze failures caused by DEPS rolls.

appengine/findit/waterfall/build_failure_analysis.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import os
 import re
 
 from common.diff import ChangeType
 from waterfall.failure_signal import FailureSignal
@@ skipping 141 matching lines @@
     if changed_src_file_path != file_path_in_log:
       hint = '%s %s (%s was in log)' % (
           change_action, changed_src_file_path, file_path_in_log)
     else:
       hint = '%s %s (and it was in log)' % (
           change_action, changed_src_file_path)
 
     self._hints[hint] += score
     self._score += score
 
+  def AddDEPSRoll(self, dep_path, dep_repo_url, dep_new_revision,
+                  dep_old_revision, file_path_in_log, score):
+    url_to_changes_in_roll = '%s/+log/%s..%s?pretty=fuller' % (
+        dep_repo_url, dep_old_revision[:12], dep_new_revision[:12])
+    hint = ('Rolled %s with changes %s (and %s was in log)' % (
+        dep_path, url_to_changes_in_roll, file_path_in_log))
+    self._hints[hint] = score
+    self._score += score
+
   def ToDict(self):
     return {
         'score': self._score,
         'hints': self._hints,
     }
 
 
 def _CheckFile(touched_file,
                file_path_in_log,
                justification,
@@ skipping 55 matching lines @@
     elif _IsRelated(changed_src_file_path, file_path_in_log):
       score = 1
 
     if score:
       justification.AddFileChange('deleted',
                                   changed_src_file_path,
                                   file_path_in_log,
                                   score,
                                   file_name_occurrences.get(file_name))
 
-def _CheckFiles(failure_signal, change_log):
+
+def _CheckFiles(failure_signal, change_log, deps_info):
   """Check files in the given change log of a CL against the failure signal.
 
   Args:
     failure_signal (FailureSignal): The failure signal of a failed step or test.
     change_log (dict): The change log of a CL as returned by
         common.change_log.ChangeLog.ToDict().
+    deps_info (dict): Output of pipeline ExtractDEPSInfoPipeline.
 
   Returns:
     A dict as returned by _Justification.ToDict() if the CL is suspected for the
     failure; otherwise None.
   """
   # Use a dict to map each file name of the touched files to their occurrences.
   file_name_occurrences = collections.defaultdict(int)
   for touched_file in change_log['touched_files']:
     change_type = touched_file['change_type']
     if (change_type in (ChangeType.ADD, ChangeType.COPY,
         ChangeType.RENAME, ChangeType.MODIFY)):
       file_name = os.path.basename(touched_file['new_path'])
       file_name_occurrences[file_name] += 1
 
     if change_type in (ChangeType.DELETE, ChangeType.RENAME):
       file_name = os.path.basename(touched_file['old_path'])
       file_name_occurrences[file_name] += 1
 
   justification = _Justification()
 
   for file_path_in_log, _ in failure_signal.files.iteritems():
-    # TODO(stgao): remove this hack when DEPS parsing is supported.
-    if file_path_in_log.startswith('src/'):
-      file_path_in_log = file_path_in_log[4:]
+    # Strip src/ from file path to make all files relative to the chromium root
+    # directory.
+    file_path_in_log = file_path_in_log.lstrip('src/')
 
     for touched_file in change_log['touched_files']:
       _CheckFile(
           touched_file, file_path_in_log, justification, file_name_occurrences)
 
+    for roll in deps_info.get('deps_rolls', {}).get(change_log['revision'], []):
+      dep_path = roll['path'].lstrip('src/')
+      if file_path_in_log.startswith(dep_path):
+        justification.AddDEPSRoll(
+            dep_path, roll['repo_url'], roll['new_revision'],
+            roll['old_revision'], file_path_in_log[len(dep_path):], 2)
+
   if not justification.score:
     return None
   else:
     return justification.ToDict()
 
 
-def AnalyzeBuildFailure(failure_info, change_logs, failure_signals):
+def AnalyzeBuildFailure(
+    failure_info, change_logs, deps_info, failure_signals):
   """Analyze the given failure signals, and figure out culprit CLs.
 
   Args:
     failure_info (dict): Output of pipeline DetectFirstFailurePipeline.
     change_logs (dict): Output of pipeline PullChangelogPipeline.
+    deps_info (dict): Output of pipeline ExtractDEPSInfoPipeline.
     failure_signals (dict): Output of pipeline ExtractSignalPipeline.
 
   Returns:
     A dict with the following form:
     {
       'failures': [
         {
           'step_name': 'compile',
           'first_failure': 230,
           'last_pass': 229,
@@ skipping 53 matching lines @@
 
     step_analysis_result = {
         'step_name': step_name,
         'first_failure': step_failure_info['first_failure'],
         'last_pass': step_failure_info.get('last_pass'),
         'suspected_cls': [],
     }
 
     while build_number <= failed_build_number:
       for revision in builds[str(build_number)]['blame_list']:
-        justification_dict = _CheckFiles(failure_signal, change_logs[revision])
+        justification_dict = _CheckFiles(
+            failure_signal, change_logs[revision], deps_info)
 
         if not justification_dict:
           continue
 
         step_analysis_result['suspected_cls'].append(
             CreateCLInfoDict(justification_dict, build_number,
                              change_logs[revision]))
 
       build_number += 1
 
     # TODO(stgao): sort CLs by score.
     analysis_result['failures'].append(step_analysis_result)
 
   return analysis_result