[Findit] Modify tryjob pipelines to trigger try jobs for test failure.

appengine/findit/waterfall/try_job_util.py

Index: appengine/findit/waterfall/try_job_util.py
diff --git a/appengine/findit/waterfall/try_job_util.py b/appengine/findit/waterfall/try_job_util.py
index b303da1dac7f8c23b50d54ecf5543ec3428a16e1..f8f58c33d11046abfe1baf2dddff51a81223255c 100644
--- a/appengine/findit/waterfall/try_job_util.py
+++ b/appengine/findit/waterfall/try_job_util.py
@@ -11,13 +11,16 @@ from model import wf_analysis_status
 from model.wf_try_job import WfTryJob
 from waterfall import try_job_pipeline
 from waterfall import waterfall_config
+from waterfall.try_job_type import TryJobType
+
 
 # TODO(chanli): Need to figure out why try-job-queue doesn't work.
 TRY_JOB_PIPELINE_QUEUE_NAME = 'build-failure-analysis-queue'
 
 
-def _CheckFailureForTryJobKey(master_name, builder_name, build_number,
-                              failure_result_map, step_name, failure):
+def _CheckFailureForTryJobKey(
+    master_name, builder_name, build_number,
+    failure_result_map, failed_step_or_test, failure):
   """Compares the current_failure and first_failure for each failed_step/test.
 
   If equal, a new try_job needs to start;
@@ -25,59 +28,93 @@ def _CheckFailureForTryJobKey(master_name, builder_name, build_number,
   """
   # TODO(chanli): Need to compare failures across builders
   # after the grouping of failures is implemented.
-  new_try_job_key = '%s/%s/%s' % (master_name, builder_name, build_number)
+  # TODO(chanli): Need to handle cases where first failure is actually
+  # more than 20 builds back. The implementation should not be here,
+  # but need to be taken care of.
   if not failure.get('last_pass'):
     # Bail out since cannot figure out the good_revision.
     return False, None
 
   if failure['current_failure'] == failure['first_failure']:
-    failure_result_map[step_name] = new_try_job_key
-    logging.info('First-time failure')
+    failure_result_map[failed_step_or_test] = '%s/%s/%s' % (
+        master_name, builder_name, build_number)
     return True, failure['last_pass']  # A new try_job is needed.
   else:
-    # TODO(chanli): Need to handle cases where first failure is actually
-    # more than 20 builds back. The implementation should not be here,
-    # but need to be taken care of.
-    try_job_key = '%s/%s/%s' % (
+    failure_result_map[failed_step_or_test] = '%s/%s/%s' % (
         master_name, builder_name, failure['first_failure'])
-    failure_result_map[step_name] = try_job_key
-    logging.info('Not first-time failure')
-    return False, failure['last_pass']
+    return False, None
+
+
+def _CheckIfNeedNewTryJobForTestFailure(
+    failure_level, master_name, builder_name, build_number,
+    failure_result_map, failures):
+  """Traverses failed steps or tests to check if a new try job is needed."""
+  need_new_try_job = False
+  last_pass = build_number
+  targeted_tests = {} if failure_level == 'step' else []
+
+  for failure_name, failure in failures.iteritems():
+    if 'tests' in failure:
+      failure_result_map[failure_name] = {}
+      failure_targeted_tests, failure_need_try_job, failure_last_pass = (
+          _CheckIfNeedNewTryJobForTestFailure(
+              'test', master_name, builder_name, build_number,
+              failure_result_map[failure_name], failure['tests']))
+      if failure_need_try_job:
+        targeted_tests[failure_name] = failure_targeted_tests
+    else:
+      failure_need_try_job, failure_last_pass = _CheckFailureForTryJobKey(
+          master_name, builder_name, build_number,
+          failure_result_map, failure_name, failure)
+      if failure_need_try_job:
+        if failure_level == 'step':
+          targeted_tests[failure_name] = []
+        else:
+          targeted_tests.append(failure_name)
+
+    need_new_try_job = need_new_try_job or failure_need_try_job
+    last_pass = (failure_last_pass if failure_last_pass and
+                 failure_last_pass < last_pass else last_pass)
+
+  return targeted_tests, need_new_try_job, last_pass
 
 
 @ndb.transactional
 def _NeedANewTryJob(
-    master_name, builder_name, build_number, failed_steps):
+    master_name, builder_name, build_number, failed_steps, failure_result_map):
   """Checks if a new try_job is needed."""
   need_new_try_job = False
-  failure_result_map = {}
-  last_pass = None
+  last_pass = build_number
 
-  for step_name, step in failed_steps.iteritems():
-    # TODO(chanli): support test failures when the recipe is ready.
-    if step_name == 'compile':
-      need_new_try_job, last_pass = _CheckFailureForTryJobKey(
-          master_name, builder_name, build_number,
-          failure_result_map, step_name, step)
-
-      if need_new_try_job:
-        try_job = WfTryJob.Get(
-            master_name, builder_name, build_number)
-
-        if try_job:
-          if try_job.failed:
-            try_job.status = wf_analysis_status.PENDING
-            try_job.put()
-          else:
-            need_new_try_job = False
-            break
-        else:
-          try_job = WfTryJob.Create(
-              master_name, builder_name, build_number)
-          try_job.put()
-      break
+  if 'compile' in failed_steps:
+    try_job_type = TryJobType.COMPILE
+    targeted_tests = None
+    need_new_try_job, last_pass = _CheckFailureForTryJobKey(
+        master_name, builder_name, build_number,
+        failure_result_map, TryJobType.COMPILE, failed_steps['compile'])
+  else:
+    try_job_type = TryJobType.TEST
+    targeted_tests, need_new_try_job, last_pass = (
+        _CheckIfNeedNewTryJobForTestFailure(
+            'step', master_name, builder_name, build_number, failure_result_map,
+            failed_steps))
 
-  return need_new_try_job, failure_result_map, last_pass
+  if need_new_try_job:
+    try_job = WfTryJob.Get(
+        master_name, builder_name, build_number)
+
+    if try_job:
+      if try_job.failed:
+        try_job.status = wf_analysis_status.PENDING
+        try_job.put()
+      else:
+        need_new_try_job = False
+    else:
+      try_job = WfTryJob.Create(
+          master_name, builder_name, build_number)
+      try_job.put()
+
+  return need_new_try_job, last_pass, try_job_type, targeted_tests
 
 
 def _GetFailedTargetsFromSignals(signals):
@@ -109,17 +146,21 @@ def ScheduleTryJobIfNeeded(failure_info, signals=None):
     logging.info('%s, %s is not supported yet.', master_name, builder_name)
     return {}
 
-  need_new_try_job, failure_result_map, last_pass = _NeedANewTryJob(
-      master_name, builder_name, build_number, failed_steps)
+  failure_result_map = {}
+  need_new_try_job, last_pass, try_job_type, targeted_tests = (
+      _NeedANewTryJob(master_name, builder_name, build_number,
+                      failed_steps, failure_result_map))
 
   if need_new_try_job:
-    compile_targets = _GetFailedTargetsFromSignals(signals)
+    compile_targets = (_GetFailedTargetsFromSignals(signals)
+                       if try_job_type == TryJobType.COMPILE else None)
 
     new_try_job_pipeline = try_job_pipeline.TryJobPipeline(
         master_name, builder_name, build_number,
         builds[str(last_pass)]['chromium_revision'],
         builds[str(build_number)]['chromium_revision'],
-        compile_targets)
+        builds[str(build_number)]['blame_list'],
+        try_job_type, compile_targets, targeted_tests)
 
     new_try_job_pipeline.target = (
         '%s.build-failure-analysis' % modules.get_current_version_name())