[Findit] Modify tryjob pipelines to trigger try jobs for test failure.

appengine/findit/waterfall/try_job_util.py

Index: appengine/findit/waterfall/try_job_util.py
diff --git a/appengine/findit/waterfall/try_job_util.py b/appengine/findit/waterfall/try_job_util.py
index b303da1dac7f8c23b50d54ecf5543ec3428a16e1..837d2392f02f1f1657f3edfbe4141ad920b2ee91 100644
--- a/appengine/findit/waterfall/try_job_util.py
+++ b/appengine/findit/waterfall/try_job_util.py
@@ -16,8 +16,9 @@ from waterfall import waterfall_config
 TRY_JOB_PIPELINE_QUEUE_NAME = 'build-failure-analysis-queue'
 
 
-def _CheckFailureForTryJobKey(master_name, builder_name, build_number,
-                              failure_result_map, step_name, failure):
+def _CheckFailureForTryJobKey(
+        master_name, builder_name, build_number,
+        failure_result_map, failed_step_or_test, failure):
   """Compares the current_failure and first_failure for each failed_step/test.
 
   If equal, a new try_job needs to start;
@@ -25,59 +26,83 @@ def _CheckFailureForTryJobKey(master_name, builder_name, build_number,
   """
   # TODO(chanli): Need to compare failures across builders
   # after the grouping of failures is implemented.
-  new_try_job_key = '%s/%s/%s' % (master_name, builder_name, build_number)
+  # TODO(chanli): Need to handle cases where first failure is actually
+  # more than 20 builds back. The implementation should not be here,
+  # but need to be taken care of.
   if not failure.get('last_pass'):
     # Bail out since cannot figure out the good_revision.
     return False, None
 
   if failure['current_failure'] == failure['first_failure']:
-    failure_result_map[step_name] = new_try_job_key
-    logging.info('First-time failure')
+    failure_result_map[failed_step_or_test] = '%s/%s/%s' % (
+        master_name, builder_name, build_number)
     return True, failure['last_pass']  # A new try_job is needed.
   else:
-    # TODO(chanli): Need to handle cases where first failure is actually
-    # more than 20 builds back. The implementation should not be here,
-    # but need to be taken care of.
-    try_job_key = '%s/%s/%s' % (
+    failure_result_map[failed_step_or_test] = '%s/%s/%s' % (
         master_name, builder_name, failure['first_failure'])
-    failure_result_map[step_name] = try_job_key
-    logging.info('Not first-time failure')
-    return False, failure['last_pass']
+    return False, None
 
 
 @ndb.transactional
 def _NeedANewTryJob(
-    master_name, builder_name, build_number, failed_steps):
+    master_name, builder_name, build_number, failed_steps, failure_result_map):
   """Checks if a new try_job is needed."""
   need_new_try_job = False
-  failure_result_map = {}
-  last_pass = None
-
-  for step_name, step in failed_steps.iteritems():
-    # TODO(chanli): support test failures when the recipe is ready.
-    if step_name == 'compile':
-      need_new_try_job, last_pass = _CheckFailureForTryJobKey(
-          master_name, builder_name, build_number,
-          failure_result_map, step_name, step)
-
-      if need_new_try_job:
-        try_job = WfTryJob.Get(
-            master_name, builder_name, build_number)
-
-        if try_job:
-          if try_job.failed:
-            try_job.status = wf_analysis_status.PENDING
-            try_job.put()
-          else:
-            need_new_try_job = False
-            break
-        else:
-          try_job = WfTryJob.Create(
-              master_name, builder_name, build_number)
-          try_job.put()
-      break
-
-  return need_new_try_job, failure_result_map, last_pass
+  last_pass = build_number
+
+  if 'compile' in failed_steps:

[lijeffrey, 2016/01/16 00:27:12]

Just to double check, compile failures and test failures will never both show up
in the same failed build?

[chanli, 2016/01/20 18:28:04]

I think I've seen cases where compile fails with some other step, but I can't
find it. But I think even if there is some other step failed, it should be
compile related. So I guess it's safe to say we can only focus on compile
failure if it happens. 

+    try_job_type = 'compile'
+    targeted_tests = None
+    need_new_try_job, last_pass = _CheckFailureForTryJobKey(
+        master_name, builder_name, build_number,
+        failure_result_map, 'compile', failed_steps['compile'])
+  else:
+    try_job_type = 'test'
+    targeted_tests = {}
+    for step_name, step in failed_steps.iteritems():
+      targeted_tests[step_name] = []
+      if 'tests' in step:
+        failure_result_map[step_name] = {}
+        step_need_new_try_job = False
+        step_last_pass = build_number
+        for test_name, test in step['tests'].iteritems():
+          test_need_new_try_job, test_last_pass = _CheckFailureForTryJobKey(
+              master_name, builder_name, build_number,
+              failure_result_map[step_name], test_name, test)
+          step_need_new_try_job = step_need_new_try_job or test_need_new_try_job
+          step_last_pass = (test_last_pass if test_last_pass and
+                   test_last_pass < step_last_pass else step_last_pass)

[lijeffrey, 2016/01/16 00:27:12]

nit: add a few spaces to lign this up with the (

[chanli, 2016/01/20 18:28:04]

Done.

+          if test_need_new_try_job:
+            targeted_tests[step_name].append(test_name)
+        if not step_need_new_try_job:
+          targeted_tests.pop(step_name)
+      else:
+        step_need_new_try_job, step_last_pass = _CheckFailureForTryJobKey(
+            master_name, builder_name, build_number,
+            failure_result_map, step_name, step)
+        if not step_need_new_try_job:
+          targeted_tests.pop(step_name)
+
+      need_new_try_job = need_new_try_job or step_need_new_try_job
+      last_pass = (step_last_pass if step_last_pass and
+                   step_last_pass < last_pass else last_pass)
+
+  if need_new_try_job:
+    try_job = WfTryJob.Get(
+        master_name, builder_name, build_number)
+
+    if try_job:
+      if try_job.failed:
+        try_job.status = wf_analysis_status.PENDING
+        try_job.put()
+      else:
+        need_new_try_job = False
+    else:
+      try_job = WfTryJob.Create(
+          master_name, builder_name, build_number)
+      try_job.put()
+
+  return need_new_try_job, last_pass, try_job_type, targeted_tests
 
 
 def _GetFailedTargetsFromSignals(signals):
@@ -109,17 +134,21 @@ def ScheduleTryJobIfNeeded(failure_info, signals=None):
     logging.info('%s, %s is not supported yet.', master_name, builder_name)
     return {}
 
-  need_new_try_job, failure_result_map, last_pass = _NeedANewTryJob(
-      master_name, builder_name, build_number, failed_steps)
+  failure_result_map = {}
+  need_new_try_job, last_pass, try_job_type, targeted_tests= (
+      _NeedANewTryJob(master_name, builder_name, build_number,
+                      failed_steps, failure_result_map))
 
   if need_new_try_job:
-    compile_targets = _GetFailedTargetsFromSignals(signals)
+    compile_targets = (_GetFailedTargetsFromSignals(signals)
+                       if try_job_type == 'compile' else None)
 
     new_try_job_pipeline = try_job_pipeline.TryJobPipeline(
         master_name, builder_name, build_number,
         builds[str(last_pass)]['chromium_revision'],
         builds[str(build_number)]['chromium_revision'],
-        compile_targets)
+        builds[str(build_number)]['blame_list'],
+        try_job_type, compile_targets, targeted_tests)
 
     new_try_job_pipeline.target = (
         '%s.build-failure-analysis' % modules.get_current_version_name())