Findit] Flake Checker: Pipeline to trigger try jobs to identify flake culprits

appengine/findit/waterfall/flake/recursive_flake_pipeline.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from datetime import timedelta
 import logging
 import random
 import textwrap
 
 from common import appengine_util
@@ skipping 37 matching lines @@
   labels = ['AnalyzedByFindit']
   comment_pipeline = PostCommentToBugPipeline(analysis.bug_id, comment, labels)
   comment_pipeline.target = appengine_util.GetTargetNameForModule(
       constants.WATERFALL_BACKEND)
   comment_pipeline.start(queue_name=queue_name)
   return True
 
 
 def _UpdateAnalysisStatusUponCompletion(
     master_flake_analysis, suspected_build, status, error):
-  master_flake_analysis.end_time = time_util.GetUTCNow()
-  master_flake_analysis.status = status
-
   if error:
     master_flake_analysis.error = error
 
-  if suspected_build != -1:
+  if suspected_build == _NO_BUILD_NUMBER:
+    master_flake_analysis.end_time = time_util.GetUTCNow()
+    master_flake_analysis.status = status
+    master_flake_analysis.result_status = result_status.NOT_FOUND_UNTRIAGED
+  else:

[chanli, 2017/01/13 00:47:33]

master_flake_analysis.status needs to be updated at all conditions. I prefer to
update it at very beginning of this function.

[lijeffrey, 2017/01/13 01:39:23]

Done.

     master_flake_analysis.suspected_flake_build_number = suspected_build

[chanli, 2017/01/13 00:47:33]

Question: should we continue triaging at build level? Especially if try jobs
don't have results?

[lijeffrey, 2017/01/13 01:39:23]

I think try jobs will be much more reliable than at the build level, because
it's easy at the build level to come up with false positives. A wrong suspected
build number is guaranteed to yield a wrong try job result if it even finds
anything. If the build level does not find anything, then that's definitely not
found (as is the case here), but if it does, it's still not necessarily found
until try jobs determine it has, especially for false positives (i.e. we suspect
a dive from 80% to 20%, but the 80% is already very flaky so I would not have
confidence in the try job finding anything meaningful.

I do plan to redesign the triage model though to differentiate between found at
the build level vs found at the CL level, but since that's not a trivial change
it should be for a separate CL.

-    master_flake_analysis.result_status = result_status.FOUND_UNTRIAGED
-  else:
-    master_flake_analysis.result_status = result_status.NOT_FOUND_UNTRIAGED
 
   master_flake_analysis.put()
 
 
 def _GetETAToStartAnalysis(manually_triggered):
   """Returns an ETA as of a UTC datetime.datetime to start the analysis.
 
   If not urgent, Swarming tasks should be run off PST peak hours from 11am to
   6pm on workdays.
 
@@ skipping 231 matching lines @@
 
   Args:
     data_points (list): A list of data points of already-completed tasks
         for this analysis. Data_points are sorted by build_numbers in descending
         order.
     flake_settings (dict): A dict of parameters for algorithms.
 
   Returns:
     (next_build_number, suspected_build): The next build number to check
         and suspected build number that the flakiness was introduced in.
-        If needs to check next_build_number, suspected_build will be -1;
-        If finds the suspected_build, next_build_number will be -1;
-        If no findings evertually, both will be -1.
+        If needs to check next_build_number, suspected_build will be
+        _NO_BUILD_NUMBER; If suspected_build is found, next_build_number will be
+        _NO_BUILD_NUMBER; If no findings eventually, both will be
+        _NO_BUILD_NUMBER.
   """
   # A description of this algorithm can be found at:
   # https://docs.google.com/document/d/1wPYFZ5OT998Yn7O8wGDOhgfcQ98mknoX13AesJaS6ig/edit
   # Get the last result.
   lower_flake_threshold = flake_settings.get('lower_flake_threshold')
   upper_flake_threshold = flake_settings.get('upper_flake_threshold')
   max_stable_in_a_row = flake_settings.get('max_stable_in_a_row')
   max_flake_in_a_row = flake_settings.get('max_flake_in_a_row')
   max_dive_in_a_row = flake_settings.get('max_dive_in_a_row')
   dive_rate_threshold = flake_settings.get('dive_rate_threshold')
@@ skipping 155 matching lines @@
         version_number, triggering_build_number,
         manually_triggered=manually_triggered,
         use_nearby_neighbor=use_nearby_neighbor,
         step_size=(current_build_number - next_build_number))
     # Disable attribute 'target' defined outside __init__ pylint warning,
     # because pipeline generates its own __init__ based on run function.
     pipeline_job.target = (  # pylint: disable=W0201
         appengine_util.GetTargetNameForModule(constants.WATERFALL_BACKEND))
     pipeline_job.StartOffPSTPeakHours(
         queue_name=self.queue_name or constants.DEFAULT_QUEUE)