[Findit] Fix 1500 byte cap on failed_steps_and_tests and output_nodes

appengine/findit/waterfall/try_job_util.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from datetime import datetime
+from datetime import timedelta
 import logging
 
 from google.appengine.ext import ndb
 
 from common import appengine_util
 from common import constants
 from common.waterfall import failure_type
 from model import analysis_status
 from model.wf_analysis import WfAnalysis
 from model.wf_build import WfBuild
 from model.wf_failure_group import WfFailureGroup
 from model.wf_try_job import WfTryJob
 from waterfall import swarming_tasks_to_try_job_pipeline
 from waterfall import waterfall_config
 from waterfall.try_job_type import TryJobType
 
 
+MATCHING_GROUPS_HOURS_AGO = 24

[lijeffrey, 2016/08/09 23:00:00]

make this seconds, but as

MATCHING_GROUPS_SECONDS_AGO = 24 * 60 * 60  # 24 hours. This gives us finer
grain control over how long ago a build failure can happen to be considered as
part of a group.

also put a TODO before it that this should be moved to config. Since your
internship is up this week put the todo under me i.e. TOOD(lijeffrey): Move this
to config.

[josiahk, 2016/08/09 23:10:36]

Done! Thanks!

+
+
 def _CheckFailureForTryJobKey(
     master_name, builder_name, build_number,
     failure_result_map, failed_step_or_test, failure):
   """Compares the current_failure and first_failure for each failed_step/test.
 
   If equal, a new try_job needs to start;
   If not, apply the key of the first_failure's try_job to this failure.
   """
   # TODO(chanli): Need to compare failures across builders
   # after the grouping of failures is implemented.
@@ skipping 157 matching lines @@
   analysis = WfAnalysis.Get(master_name, builder_name, build_number)
   analysis.failure_group_key = failure_group_key
   analysis.put()
 
 
 def _CreateBuildFailureGroup(
     master_name, builder_name, build_number, build_failure_type, blame_list,
     suspected_tuples, output_nodes=None, failed_steps_and_tests=None):
   new_group = WfFailureGroup.Create(master_name, builder_name, build_number)
   new_group.build_failure_type = build_failure_type
+  new_group.created_time = datetime.utcnow()

[lijeffrey, 2016/08/09 23:00:00]

nit: you should move created time to the first call before updating
build_failure_type

[josiahk, 2016/08/09 23:10:36]

Done.

   new_group.blame_list = blame_list
   new_group.suspected_tuples = suspected_tuples
   new_group.output_nodes = output_nodes
   new_group.failed_steps_and_tests = failed_steps_and_tests
   new_group.put()
 
 
 def _GetMatchingGroup(wf_failure_groups, blame_list, suspected_tuples):
   for group in wf_failure_groups:
     if _BlameListsIntersection(group.blame_list, blame_list):
       if suspected_tuples == group.suspected_tuples:
         return group
 
   return None
 
 
 def _GetOutputNodes(signals):
   if not signals or 'compile' not in signals:
     return []
 
   # Compile failures with no output nodes will be considered unique.
   return signals['compile'].get('failed_output_nodes', [])
 
 
+def _GetMatchingFailureGroups(build_failure_type):
+  earliest_time = datetime.utcnow() - timedelta(hours=MATCHING_GROUPS_HOURS_AGO)
+  return WfFailureGroup.query(ndb.AND(
+      WfFailureGroup.created_time >= earliest_time,
+      WfFailureGroup.build_failure_type == build_failure_type)).fetch()
+
+
 def _GetMatchingCompileFailureGroups(output_nodes):
+  groups = _GetMatchingFailureGroups(failure_type.COMPILE)
   # Output nodes should already be unique and sorted.
-  return WfFailureGroup.query(ndb.AND(
-      WfFailureGroup.build_failure_type == failure_type.COMPILE,
-      WfFailureGroup.output_nodes == output_nodes
-  )).fetch()
+  return [group for group in groups if group.output_nodes == output_nodes]
 
 
 def _GetMatchingTestFailureGroups(failed_steps_and_tests):
-  return WfFailureGroup.query(ndb.AND(
-      WfFailureGroup.build_failure_type == failure_type.TEST,
-      WfFailureGroup.failed_steps_and_tests == failed_steps_and_tests
-  )).fetch()
+  groups = _GetMatchingFailureGroups(failure_type.TEST)
+  return [group for group in groups
+          if group.failed_steps_and_tests == failed_steps_and_tests]
 
 
 def _IsBuildFailureUniqueAcrossPlatforms(
     master_name, builder_name, build_number, build_failure_type, blame_list,
     failed_steps, signals, heuristic_result):
   output_nodes = None
   failed_steps_and_tests = None
 
   if build_failure_type == failure_type.COMPILE:
     output_nodes = _GetOutputNodes(signals)
@@ skipping 64 matching lines @@
     need_new_try_job, last_pass = _CheckFailureForTryJobKey(
         master_name, builder_name, build_number,
         failure_result_map, TryJobType.COMPILE, failed_steps['compile'])
   else:
     try_job_type = TryJobType.TEST
     targeted_tests, need_new_try_job, last_pass = (
         _CheckIfNeedNewTryJobForTestFailure(
             'step', master_name, builder_name, build_number, failure_result_map,
             failed_steps))
 
-
   need_new_try_job = (
       need_new_try_job and ReviveOrCreateTryJobEntity(
           master_name, builder_name, build_number, force_try_job))
 
   # TODO(josiahk): Integrate _IsBuildFailureUniqueAcrossPlatforms() into
   # need_new_try_job boolean
   if need_new_try_job:
     _IsBuildFailureUniqueAcrossPlatforms(
         master_name, builder_name, build_number, build_failure_type,
         builds[str(build_number)]['blame_list'], failed_steps, signals,
@@ skipping 102 matching lines @@
               pipeline.pipeline_status_path, try_job_type)
     else:  # pragma: no cover
       logging_str = (
           'Try job was scheduled for build %s, %s, %s: %s because of %s '
           'failure.') % (
               master_name, builder_name, build_number,
               pipeline.pipeline_status_path, try_job_type)
     logging.info(logging_str)
 
   return failure_result_map