[Findit] Add three sub-pipelines to analyze build failure.

appengine/findit/waterfall/build_failure_analysis_pipelines.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from datetime import datetime
 import logging
 
 from google.appengine.ext import ndb
 
 from model.build_analysis import BuildAnalysis
 from model.build_analysis_status import BuildAnalysisStatus
 from waterfall.base_pipeline import BasePipeline
 from waterfall.detect_first_failure_pipeline import DetectFirstFailurePipeline
+from waterfall.extract_signal_pipeline import ExtractSignalPipeline
+from waterfall.identify_culprit_pipeline import IdentifyCulpritPipeline
+from waterfall.pull_changelog_pipeline import PullChangelogPipeline
 
 
 class BuildFailurePipeline(BasePipeline):
 
   def __init__(self, master_name, builder_name, build_number):
     super(BuildFailurePipeline, self).__init__(
         master_name, builder_name, build_number)
     self.master_name = master_name
     self.builder_name = builder_name
     self.build_number = build_number
 
   def finalized(self):

[qyearsley, 2015/01/15 19:10:28]

When is this method called? Why is no BuildAnalysis created/put when was_aborted
is False? Is it created and put at some other time?

[stgao, 2015/01/16 20:21:39]

This method is called when all sub-pipelines finish running.
If not aborted, the analysis entity is updated in identify_culprit_pipeline.py,
so there is no need to update here.
The analysis is created before the pipeline starts.

I also added a comment in the code to explain this.

-    analysis = BuildAnalysis.GetBuildAnalysis(
-        self.master_name, self.builder_name, self.build_number)
     if self.was_aborted:  # pragma: no cover
+      analysis = BuildAnalysis.GetBuildAnalysis(
+          self.master_name, self.builder_name, self.build_number)
       analysis.status = BuildAnalysisStatus.ERROR
-    else:
-      analysis.status = BuildAnalysisStatus.ANALYZED
-    analysis.put()
+      analysis.put()
 
   # Arguments number differs from overridden method - pylint: disable=W0221
   def run(self, master_name, builder_name, build_number):
     analysis = BuildAnalysis.GetBuildAnalysis(
         master_name, builder_name, build_number)
     analysis.pipeline_url = self.pipeline_status_url()
     analysis.status = BuildAnalysisStatus.ANALYZING
     analysis.start_time = datetime.utcnow()
     analysis.put()
 
-    yield DetectFirstFailurePipeline(master_name, builder_name, build_number)
+    failure_info = yield DetectFirstFailurePipeline(

[qyearsley, 2015/01/15 19:10:28]

It may be worth adding a comment here saying what failure_info will be set to...
not sure if there's any Pipeline documentation related to this?

[stgao, 2015/01/16 20:21:39]

Done.
I added a comment.

Yes, the documentation for the yield is
https://github.com/GoogleCloudPlatform/appengine-pipelines/wiki/Python#defini...

+        master_name, builder_name, build_number)
+    change_logs = yield PullChangelogPipeline(failure_info)
+    signals = yield ExtractSignalPipeline(failure_info)
+    yield IdentifyCulpritPipeline(failure_info, change_logs, signals)
 
 
 @ndb.transactional
 def NeedANewAnalysis(master_name, builder_name, build_number, force):
   """Checks status of analysis for the build and decides if a new one is needed.
 
   A BuildAnalysis entity for the given build will be created if none exists.
 
   Returns:
     True if an analysis is needed, otherwise False.
@@ skipping 28 matching lines @@
     pipeline_job = BuildFailurePipeline(master_name, builder_name, build_number)
     pipeline_job.start(queue_name=queue_name)
 
     logging.info('An analysis triggered on build %s, %s, %s: %s',
                  master_name, builder_name, build_number,
                  pipeline_job.pipeline_status_url())
   else:  # pragma: no cover
     logging.info('Analysis was already triggered or the result is recent.')
 
   return BuildAnalysis.GetBuildAnalysis(master_name, builder_name, build_number)