[Findit] Modify tryjob pipelines to trigger try jobs for test failure.

appengine/findit/waterfall/try_job_util.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 
 from google.appengine.api import modules
 from google.appengine.ext import ndb
 
 from model import wf_analysis_status
 from model.wf_try_job import WfTryJob
 from waterfall import try_job_pipeline
 from waterfall import waterfall_config
 
 # TODO(chanli): Need to figure out why try-job-queue doesn't work.
 TRY_JOB_PIPELINE_QUEUE_NAME = 'build-failure-analysis-queue'
 
 
-def _CheckFailureForTryJobKey(master_name, builder_name, build_number,
-                              failure_result_map, step_name, failure):
+def _CheckFailureForTryJobKey(
+    master_name, builder_name, build_number,
+    failure_result_map, failed_step_or_test, failure):
   """Compares the current_failure and first_failure for each failed_step/test.
 
   If equal, a new try_job needs to start;
   If not, apply the key of the first_failure's try_job to this failure.
   """
   # TODO(chanli): Need to compare failures across builders
   # after the grouping of failures is implemented.
-  new_try_job_key = '%s/%s/%s' % (master_name, builder_name, build_number)
+  # TODO(chanli): Need to handle cases where first failure is actually
+  # more than 20 builds back. The implementation should not be here,
+  # but need to be taken care of.
   if not failure.get('last_pass'):
     # Bail out since cannot figure out the good_revision.
     return False, None
 
   if failure['current_failure'] == failure['first_failure']:
-    failure_result_map[step_name] = new_try_job_key
-    logging.info('First-time failure')
+    failure_result_map[failed_step_or_test] = '%s/%s/%s' % (
+        master_name, builder_name, build_number)
     return True, failure['last_pass']  # A new try_job is needed.
   else:
-    # TODO(chanli): Need to handle cases where first failure is actually
-    # more than 20 builds back. The implementation should not be here,
-    # but need to be taken care of.
-    try_job_key = '%s/%s/%s' % (
+    failure_result_map[failed_step_or_test] = '%s/%s/%s' % (
         master_name, builder_name, failure['first_failure'])
-    failure_result_map[step_name] = try_job_key
-    logging.info('Not first-time failure')
-    return False, failure['last_pass']
+    return False, None
 
 
 @ndb.transactional
 def _NeedANewTryJob(
-    master_name, builder_name, build_number, failed_steps):
+    master_name, builder_name, build_number, failed_steps, failure_result_map):
   """Checks if a new try_job is needed."""
   need_new_try_job = False
-  failure_result_map = {}
-  last_pass = None
+  last_pass = build_number
 
-  for step_name, step in failed_steps.iteritems():
-    # TODO(chanli): support test failures when the recipe is ready.
-    if step_name == 'compile':
-      need_new_try_job, last_pass = _CheckFailureForTryJobKey(
-          master_name, builder_name, build_number,
-          failure_result_map, step_name, step)
+  if 'compile' in failed_steps:
+    try_job_type = 'compile'
+    targeted_tests = None
+    need_new_try_job, last_pass = _CheckFailureForTryJobKey(
+        master_name, builder_name, build_number,
+        failure_result_map, 'compile', failed_steps['compile'])
+  else:
+    try_job_type = 'test'
+    targeted_tests = {}
+    for step_name, step in failed_steps.iteritems():
+      targeted_tests[step_name] = []
+      if 'tests' in step:
+        failure_result_map[step_name] = {}
+        step_need_new_try_job = False
+        step_last_pass = build_number
+        for test_name, test in step['tests'].iteritems():
+          test_need_new_try_job, test_last_pass = _CheckFailureForTryJobKey(
+              master_name, builder_name, build_number,
+              failure_result_map[step_name], test_name, test)
+          step_need_new_try_job = step_need_new_try_job or test_need_new_try_job
+          step_last_pass = (test_last_pass if test_last_pass and
+                   test_last_pass < step_last_pass else step_last_pass)
+          if test_need_new_try_job:
+            targeted_tests[step_name].append(test_name)
+        if not step_need_new_try_job:
+          targeted_tests.pop(step_name)
+      else:
+        step_need_new_try_job, step_last_pass = _CheckFailureForTryJobKey(
+            master_name, builder_name, build_number,
+            failure_result_map, step_name, step)
+        if not step_need_new_try_job:
+          targeted_tests.pop(step_name)
 
-      if need_new_try_job:
-        try_job = WfTryJob.Get(
-            master_name, builder_name, build_number)
+      need_new_try_job = need_new_try_job or step_need_new_try_job
+      last_pass = (step_last_pass if step_last_pass and
+                   step_last_pass < last_pass else last_pass)
 
-        if try_job:
-          if try_job.failed:
-            try_job.status = wf_analysis_status.PENDING
-            try_job.put()
-          else:
-            need_new_try_job = False
-            break
-        else:
-          try_job = WfTryJob.Create(
-              master_name, builder_name, build_number)
-          try_job.put()
-      break
+  if need_new_try_job:
+    try_job = WfTryJob.Get(
+        master_name, builder_name, build_number)
 
-  return need_new_try_job, failure_result_map, last_pass
+    if try_job:
+      if try_job.failed:
+        try_job.status = wf_analysis_status.PENDING
+        try_job.put()
+      else:
+        need_new_try_job = False
+    else:
+      try_job = WfTryJob.Create(
+          master_name, builder_name, build_number)
+      try_job.put()
+
+  return need_new_try_job, last_pass, try_job_type, targeted_tests
 
 
 def _GetFailedTargetsFromSignals(signals):
   compile_targets = []
 
   if not signals or 'compile' not in signals:
     return compile_targets
 
   for source_target in signals['compile']['failed_targets']:
     # Link failures have only targets but no source. TODO(lijeffrey):
@@ skipping 11 matching lines @@
   build_number = failure_info['build_number']
   failed_steps = failure_info.get('failed_steps', [])
   builds = failure_info.get('builds', {})
 
   tryserver_mastername, tryserver_buildername = (
       waterfall_config.GetTrybotForWaterfallBuilder(master_name, builder_name))
   if not tryserver_mastername or not tryserver_buildername:
     logging.info('%s, %s is not supported yet.', master_name, builder_name)
     return {}
 
-  need_new_try_job, failure_result_map, last_pass = _NeedANewTryJob(
-      master_name, builder_name, build_number, failed_steps)
+  failure_result_map = {}
+  need_new_try_job, last_pass, try_job_type, targeted_tests= (

[qyearsley, 2016/01/17 03:43:12]

space before =

[chanli, 2016/01/20 18:07:09]

Done.

+      _NeedANewTryJob(master_name, builder_name, build_number,
+                      failed_steps, failure_result_map))
 
   if need_new_try_job:
-    compile_targets = _GetFailedTargetsFromSignals(signals)
+    compile_targets = (_GetFailedTargetsFromSignals(signals)
+                       if try_job_type == 'compile' else None)
 
     new_try_job_pipeline = try_job_pipeline.TryJobPipeline(
         master_name, builder_name, build_number,
         builds[str(last_pass)]['chromium_revision'],
         builds[str(build_number)]['chromium_revision'],
-        compile_targets)
+        builds[str(build_number)]['blame_list'],
+        try_job_type, compile_targets, targeted_tests)
 
     new_try_job_pipeline.target = (
         '%s.build-failure-analysis' % modules.get_current_version_name())
     new_try_job_pipeline.start(queue_name=TRY_JOB_PIPELINE_QUEUE_NAME)
     logging.info('Try-job was scheduled for build %s, %s, %s: %s',
                  master_name, builder_name, build_number,
                  new_try_job_pipeline.pipeline_status_path)
 
   return failure_result_map