[Findit] Fix 1500 byte cap on failed_steps_and_tests and output_nodes

appengine/findit/waterfall/try_job_util.py

Index: appengine/findit/waterfall/try_job_util.py
diff --git a/appengine/findit/waterfall/try_job_util.py b/appengine/findit/waterfall/try_job_util.py
index eaa4209bf663a6f600d080d825e16ffdbfdb6733..acb16fbd705747c23a8cc7457b21743aba188aed 100644
--- a/appengine/findit/waterfall/try_job_util.py
+++ b/appengine/findit/waterfall/try_job_util.py
@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 from datetime import datetime
+from datetime import timedelta
 import logging
 
 from google.appengine.ext import ndb
@@ -20,6 +21,9 @@ from waterfall import waterfall_config
 from waterfall.try_job_type import TryJobType
 
 
+MATCHING_GROUPS_HOURS_AGO = 24

[lijeffrey, 2016/08/09 23:00:00]

make this seconds, but as

MATCHING_GROUPS_SECONDS_AGO = 24 * 60 * 60  # 24 hours. This gives us finer
grain control over how long ago a build failure can happen to be considered as
part of a group.

also put a TODO before it that this should be moved to config. Since your
internship is up this week put the todo under me i.e. TOOD(lijeffrey): Move this
to config.

[josiahk, 2016/08/09 23:10:36]

Done! Thanks!

+
+
 def _CheckFailureForTryJobKey(
     master_name, builder_name, build_number,
     failure_result_map, failed_step_or_test, failure):
@@ -197,6 +201,7 @@ def _CreateBuildFailureGroup(
     suspected_tuples, output_nodes=None, failed_steps_and_tests=None):
   new_group = WfFailureGroup.Create(master_name, builder_name, build_number)
   new_group.build_failure_type = build_failure_type
+  new_group.created_time = datetime.utcnow()

[lijeffrey, 2016/08/09 23:00:00]

nit: you should move created time to the first call before updating
build_failure_type

[josiahk, 2016/08/09 23:10:36]

Done.

   new_group.blame_list = blame_list
   new_group.suspected_tuples = suspected_tuples
   new_group.output_nodes = output_nodes
@@ -221,19 +226,23 @@ def _GetOutputNodes(signals):
   return signals['compile'].get('failed_output_nodes', [])
 
 
+def _GetMatchingFailureGroups(build_failure_type):
+  earliest_time = datetime.utcnow() - timedelta(hours=MATCHING_GROUPS_HOURS_AGO)
+  return WfFailureGroup.query(ndb.AND(
+      WfFailureGroup.created_time >= earliest_time,
+      WfFailureGroup.build_failure_type == build_failure_type)).fetch()
+
+
 def _GetMatchingCompileFailureGroups(output_nodes):
+  groups = _GetMatchingFailureGroups(failure_type.COMPILE)
   # Output nodes should already be unique and sorted.
-  return WfFailureGroup.query(ndb.AND(
-      WfFailureGroup.build_failure_type == failure_type.COMPILE,
-      WfFailureGroup.output_nodes == output_nodes
-  )).fetch()
+  return [group for group in groups if group.output_nodes == output_nodes]
 
 
 def _GetMatchingTestFailureGroups(failed_steps_and_tests):
-  return WfFailureGroup.query(ndb.AND(
-      WfFailureGroup.build_failure_type == failure_type.TEST,
-      WfFailureGroup.failed_steps_and_tests == failed_steps_and_tests
-  )).fetch()
+  groups = _GetMatchingFailureGroups(failure_type.TEST)
+  return [group for group in groups
+          if group.failed_steps_and_tests == failed_steps_and_tests]
 
 
 def _IsBuildFailureUniqueAcrossPlatforms(
@@ -318,7 +327,6 @@ def _NeedANewTryJob(
             'step', master_name, builder_name, build_number, failure_result_map,
             failed_steps))
 
-
   need_new_try_job = (
       need_new_try_job and ReviveOrCreateTryJobEntity(
           master_name, builder_name, build_number, force_try_job))