[Findit] Modify tryjob pipelines to trigger try jobs for test failure.

appengine/findit/waterfall/try_job_util.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 
 from google.appengine.api import modules
 from google.appengine.ext import ndb
 
 from model import wf_analysis_status
 from model.wf_try_job import WfTryJob
+from waterfall.try_job_enums import TryJobType
 from waterfall import try_job_pipeline
 from waterfall import waterfall_config
 
+
 # TODO(chanli): Need to figure out why try-job-queue doesn't work.
 TRY_JOB_PIPELINE_QUEUE_NAME = 'build-failure-analysis-queue'
 
 
-def _CheckFailureForTryJobKey(master_name, builder_name, build_number,
-                              failure_result_map, step_name, failure):
+def _CheckFailureForTryJobKey(
+    master_name, builder_name, build_number,
+    failure_result_map, failed_step_or_test, failure):
   """Compares the current_failure and first_failure for each failed_step/test.
 
   If equal, a new try_job needs to start;
   If not, apply the key of the first_failure's try_job to this failure.
   """
   # TODO(chanli): Need to compare failures across builders
   # after the grouping of failures is implemented.
-  new_try_job_key = '%s/%s/%s' % (master_name, builder_name, build_number)
+  # TODO(chanli): Need to handle cases where first failure is actually
+  # more than 20 builds back. The implementation should not be here,
+  # but need to be taken care of.
   if not failure.get('last_pass'):
     # Bail out since cannot figure out the good_revision.
     return False, None
 
   if failure['current_failure'] == failure['first_failure']:
-    failure_result_map[step_name] = new_try_job_key
-    logging.info('First-time failure')
+    failure_result_map[failed_step_or_test] = '%s/%s/%s' % (
+        master_name, builder_name, build_number)
     return True, failure['last_pass']  # A new try_job is needed.
   else:
-    # TODO(chanli): Need to handle cases where first failure is actually
-    # more than 20 builds back. The implementation should not be here,
-    # but need to be taken care of.
-    try_job_key = '%s/%s/%s' % (
+    failure_result_map[failed_step_or_test] = '%s/%s/%s' % (
         master_name, builder_name, failure['first_failure'])
-    failure_result_map[step_name] = try_job_key
-    logging.info('Not first-time failure')
-    return False, failure['last_pass']
+    return False, None
+
+
+def _CheckIfNeedNewTryJobForTestFailure(
+    failure_level, master_name, builder_name, build_number,
+    failure_result_map, failures):
+  """Traverses failed steps or tests to check if a new try job is needed."""
+  need_new_try_job = False
+  last_pass = build_number
+  targeted_tests = {} if failure_level == 'step' else []
+
+  for failure_name, failure in failures.iteritems():
+    if failure_level == 'step':
+      targeted_tests[failure_name] = []
+    else:
+      targeted_tests.append(failure_name)
+
+    if 'tests' in failure:
+      failure_result_map[failure_name] = {}
+      targeted_tests[failure_name], failure_need_try_job, failure_last_pass =(
+          _CheckIfNeedNewTryJobForTestFailure(
+              'test', master_name, builder_name, build_number,
+              failure_result_map[failure_name], failure['tests']))
+      if not failure_need_try_job:
+        targeted_tests.pop(failure_name)
+    else:
+      failure_need_try_job, failure_last_pass = _CheckFailureForTryJobKey(
+          master_name, builder_name, build_number,
+          failure_result_map, failure_name, failure)
+      if not failure_need_try_job:
+        if failure_level == 'step':
+          targeted_tests.pop(failure_name)
+        else:
+          targeted_tests.remove(failure_name)
+
+    need_new_try_job = need_new_try_job or failure_need_try_job
+    last_pass = (failure_last_pass if failure_last_pass and
+                 failure_last_pass < last_pass else last_pass)
+
+  return targeted_tests, need_new_try_job, last_pass
 
 
 @ndb.transactional
 def _NeedANewTryJob(
-    master_name, builder_name, build_number, failed_steps):
+    master_name, builder_name, build_number, failed_steps, failure_result_map):
   """Checks if a new try_job is needed."""
   need_new_try_job = False
-  failure_result_map = {}
-  last_pass = None
+  last_pass = build_number
 
-  for step_name, step in failed_steps.iteritems():
-    # TODO(chanli): support test failures when the recipe is ready.
-    if step_name == 'compile':
-      need_new_try_job, last_pass = _CheckFailureForTryJobKey(
-          master_name, builder_name, build_number,
-          failure_result_map, step_name, step)
+  if 'compile' in failed_steps:
+    try_job_type = TryJobType.type_compile
+    targeted_tests = None
+    need_new_try_job, last_pass = _CheckFailureForTryJobKey(
+        master_name, builder_name, build_number,
+        failure_result_map, TryJobType.type_compile, failed_steps['compile'])
+  else:
+    try_job_type = TryJobType.type_test
+    targeted_tests, need_new_try_job, last_pass =(
+        _CheckIfNeedNewTryJobForTestFailure(
+            'step', master_name, builder_name, build_number, failure_result_map,
+            failed_steps))
 
-      if need_new_try_job:
-        try_job = WfTryJob.Get(
-            master_name, builder_name, build_number)
+  if need_new_try_job:
+    try_job = WfTryJob.Get(
+        master_name, builder_name, build_number)
 
-        if try_job:
-          if try_job.failed:
-            try_job.status = wf_analysis_status.PENDING
-            try_job.put()
-          else:
-            need_new_try_job = False
-            break
-        else:
-          try_job = WfTryJob.Create(
-              master_name, builder_name, build_number)
-          try_job.put()
-      break
+    if try_job:
+      if try_job.failed:
+        try_job.status = wf_analysis_status.PENDING
+        try_job.put()
+      else:
+        need_new_try_job = False
+    else:
+      try_job = WfTryJob.Create(
+          master_name, builder_name, build_number)
+      try_job.put()
 
-  return need_new_try_job, failure_result_map, last_pass
+  return need_new_try_job, last_pass, try_job_type, targeted_tests
 
 
 def _GetFailedTargetsFromSignals(signals):
   compile_targets = []
 
   if not signals or 'compile' not in signals:
     return compile_targets
 
   for source_target in signals['compile']['failed_targets']:
     # Link failures have only targets but no source. TODO(lijeffrey):
@@ skipping 11 matching lines @@
   build_number = failure_info['build_number']
   failed_steps = failure_info.get('failed_steps', [])
   builds = failure_info.get('builds', {})
 
   tryserver_mastername, tryserver_buildername = (
       waterfall_config.GetTrybotForWaterfallBuilder(master_name, builder_name))
   if not tryserver_mastername or not tryserver_buildername:
     logging.info('%s, %s is not supported yet.', master_name, builder_name)
     return {}
 
-  need_new_try_job, failure_result_map, last_pass = _NeedANewTryJob(
-      master_name, builder_name, build_number, failed_steps)
+  failure_result_map = {}
+  need_new_try_job, last_pass, try_job_type, targeted_tests = (
+      _NeedANewTryJob(master_name, builder_name, build_number,
+                      failed_steps, failure_result_map))
 
   if need_new_try_job:
-    compile_targets = _GetFailedTargetsFromSignals(signals)
+    compile_targets = (_GetFailedTargetsFromSignals(signals)
+                       if try_job_type == TryJobType.type_compile else None)
 
     new_try_job_pipeline = try_job_pipeline.TryJobPipeline(
         master_name, builder_name, build_number,
         builds[str(last_pass)]['chromium_revision'],
         builds[str(build_number)]['chromium_revision'],
-        compile_targets)
+        builds[str(build_number)]['blame_list'],
+        try_job_type, compile_targets, targeted_tests)
 
     new_try_job_pipeline.target = (
         '%s.build-failure-analysis' % modules.get_current_version_name())
     new_try_job_pipeline.start(queue_name=TRY_JOB_PIPELINE_QUEUE_NAME)
     logging.info('Try-job was scheduled for build %s, %s, %s: %s',
                  master_name, builder_name, build_number,
                  new_try_job_pipeline.pipeline_status_path)
 
   return failure_result_map