[Findit] Adding logic to force try jobs regardless of bailout or previous results

appengine/findit/waterfall/try_job_util.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from datetime import datetime
 import logging
 
 from google.appengine.ext import ndb
 
 from common import appengine_util
@@ skipping 63 matching lines @@
 
     need_new_try_job = need_new_try_job or failure_need_try_job
     last_pass = (failure_last_pass if failure_last_pass and
                  failure_last_pass < last_pass else last_pass)
 
   return targeted_tests, need_new_try_job, last_pass
 
 
 @ndb.transactional
 def _NeedANewTryJob(
-    master_name, builder_name, build_number, failed_steps, failure_result_map):
+    master_name, builder_name, build_number, failed_steps, failure_result_map,
+    force_try_job=False):
   """Checks if a new try_job is needed."""
   need_new_try_job = False
   last_pass = build_number
 
   if 'compile' in failed_steps:
     try_job_type = TryJobType.COMPILE
     targeted_tests = None
     need_new_try_job, last_pass = _CheckFailureForTryJobKey(
         master_name, builder_name, build_number,
         failure_result_map, TryJobType.COMPILE, failed_steps['compile'])
   else:
     try_job_type = TryJobType.TEST
     targeted_tests, need_new_try_job, last_pass = (
         _CheckIfNeedNewTryJobForTestFailure(
             'step', master_name, builder_name, build_number, failure_result_map,
             failed_steps))
 
   if need_new_try_job:
     try_job = WfTryJob.Get(master_name, builder_name, build_number)
 
     if try_job:
-      if try_job.failed:
+      if try_job.failed or force_try_job:
         try_job.status = analysis_status.PENDING
         try_job.put()
       else:
         need_new_try_job = False
     else:
       try_job = WfTryJob.Create(master_name, builder_name, build_number)
       try_job.put()
 
   return need_new_try_job, last_pass, try_job_type, targeted_tests
 
@@ skipping 25 matching lines @@
   if not heuristic_result:
     return list(suspected_revisions)
   for failure in heuristic_result.get('failures', []):
     for cl in failure['suspected_cls']:
       suspected_revisions.add(cl['revision'])
   return list(suspected_revisions)
 
 
 def _ShouldBailOutForOutdatedBuild(build):
   return (datetime.utcnow() - build.start_time).days > 0
-    
 
-def ScheduleTryJobIfNeeded(failure_info, signals, heuristic_result):
+
+def ScheduleTryJobIfNeeded(failure_info, signals, heuristic_result,
+                           force_try_job=False):
   master_name = failure_info['master_name']
   builder_name = failure_info['builder_name']
   build_number = failure_info['build_number']
   failed_steps = failure_info.get('failed_steps', [])
   builds = failure_info.get('builds', {})
 
-  # Bail out if the build data's timestamp is more than 24 hours old to
-  # avoid using outdated revisions. TODO(lijeffrey): This will also disallow
-  # manually triggering try jobs more than a day old using build_completed=1.
-  # Need to implement a flag to force try jobs regardless of timestamp.
-  build = WfBuild.Get(master_name, builder_name, build_number)
-  if _ShouldBailOutForOutdatedBuild(build):
-    logging.error(
-        'Build time is more than 24 hours old. Try job will not be triggered.')
-    return {}
-
   tryserver_mastername, tryserver_buildername = (
       waterfall_config.GetTrybotForWaterfallBuilder(master_name, builder_name))
 
   if not tryserver_mastername or not tryserver_buildername:
     logging.info('%s, %s is not supported yet.', master_name, builder_name)
     return {}
-  elif (failure_info['failure_type'] == failure_type.TEST and
+
+  if not force_try_job:
+    build = WfBuild.Get(master_name, builder_name, build_number)
+
+    if _ShouldBailOutForOutdatedBuild(build):
+      logging.error('Build time %s is more than 24 hours old. '
+                    'Try job will not be triggered.' % build.start_time)
+      return {}
+
+    if (failure_info['failure_type'] == failure_type.TEST and
         waterfall_config.ShouldSkipTestTryJobs(master_name, builder_name)):
-    logging.info('Test try jobs on %s, %s are not supported yet.',
-                 master_name, builder_name)
-    return {}
+      logging.info('Test try jobs on %s, %s are not supported yet.',
+                   master_name, builder_name)
+      return {}
 
   failure_result_map = {}
   need_new_try_job, last_pass, try_job_type, targeted_tests = (
       _NeedANewTryJob(master_name, builder_name, build_number,
-                      failed_steps, failure_result_map))
+                      failed_steps, failure_result_map, force_try_job))
 
   if need_new_try_job:
     compile_targets = (_GetFailedTargetsFromSignals(
         signals, master_name, builder_name)
                        if try_job_type == TryJobType.COMPILE else None)
     suspected_revisions = _GetSuspectsFromHeuristicResult(heuristic_result)
 
     pipeline = (
         swarming_tasks_to_try_job_pipeline.SwarmingTasksToTryJobPipeline(
             master_name, builder_name, build_number,
             builds[str(last_pass)]['chromium_revision'],
             builds[str(build_number)]['chromium_revision'],
             builds[str(build_number)]['blame_list'],
-            try_job_type, compile_targets, targeted_tests, suspected_revisions))
+            try_job_type, compile_targets, targeted_tests, suspected_revisions,
+            force_try_job))
 
     pipeline.target = appengine_util.GetTargetNameForModule(
         constants.WATERFALL_BACKEND)
     pipeline.start(queue_name=constants.WATERFALL_TRY_JOB_QUEUE)
 
     if try_job_type == TryJobType.TEST:  # pragma: no cover
       logging_str = (
           'Trying to schedule swarming task(s) for build %s, %s, %s: %s'
           ' because of %s failure. A try job may be triggered if some reliable'
           ' failure is detected in task(s).') % (
               master_name, builder_name, build_number,
               pipeline.pipeline_status_path, try_job_type)
     else:  # pragma: no cover
       logging_str = (
           'Try job was scheduled for build %s, %s, %s: %s because of %s '
           'failure.') % (
               master_name, builder_name, build_number,
               pipeline.pipeline_status_path, try_job_type)
     logging.info(logging_str)
 
   return failure_result_map