[Findit] Compare lists to lists instead of lists to tuples, and change tests

appengine/findit/waterfall/try_job_util.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from datetime import datetime
 import logging
 
 from google.appengine.ext import ndb
 
 from common import appengine_util
@@ skipping 126 matching lines @@
     return failed_steps_and_tests
 
   for failed_step_name, failed_step in failed_steps.iteritems():
     failed_steps_and_tests[failed_step_name] = []
     for test_name in failed_step.get('tests', {}):
       failed_steps_and_tests[failed_step_name].append(test_name)
 
   return failed_steps_and_tests
 
 
-def GenPotentialCulpritTupleList(heuristic_result):
-  """Generates a list of potential culprit tuples.
+def GetSuspectedCLsWithFailures(heuristic_result):
+  """Generates a list of suspected CLs with failures.
 
   Args:
-    heuristic_result: the heuristic_result from which to generate a potential
-        culprit tuple list.
+    heuristic_result: the heuristic_result from which to generate the list of
+    suspected CLs with failures.
 
   Returns:
-    A list of cultprit tuples that each could look like:
+    A list of suspected CLs with failures that each could look like:
 
-        (step_name, revision, test_name)
+        [step_name, revision, test_name]
 
     or could look like:
 
-        (step_name, revision, None)
+        [step_name, revision, None]
   """
-  potential_culprit_tuple_list = []
+  suspected_cls_with_failures = []
 
   if not heuristic_result:
-    return potential_culprit_tuple_list
+    return suspected_cls_with_failures
 
-  # Iterates through the failures, tests, and suspected_cls, appending potential
-  # (step_name, test_name, revision) and (step_name, revision) culprit tuples to
-  # the list.
+  # Iterates through the failures, tests, and suspected_cls, appending suspected
+  # CLs and failures to the list.
   for failure in heuristic_result['failures']:
     if failure.get('tests'):
       for test in failure['tests']:
         for suspected_cl in test.get('suspected_cls', []):
-          potential_culprit_tuple_list.append((
+          suspected_cls_with_failures.append([
               failure['step_name'],
               suspected_cl['revision'],
-              test['test_name']))
+              test['test_name']])
     else:
       for suspected_cl in failure['suspected_cls']:
-        potential_culprit_tuple_list.append((
+        suspected_cls_with_failures.append([
             failure['step_name'],
             suspected_cl['revision'],
-            None))
+            None])
 
-  return potential_culprit_tuple_list
+  return suspected_cls_with_failures
 
 
 def _LinkAnalysisToBuildFailureGroup(
     master_name, builder_name, build_number, failure_group_key):
   analysis = WfAnalysis.Get(master_name, builder_name, build_number)
   analysis.failure_group_key = failure_group_key
   analysis.put()
 
 
 def _CreateBuildFailureGroup(
     master_name, builder_name, build_number, build_failure_type, blame_list,
     suspected_tuples, output_nodes=None, failed_steps_and_tests=None):
   new_group = WfFailureGroup.Create(master_name, builder_name, build_number)
   new_group.build_failure_type = build_failure_type
   new_group.blame_list = blame_list
   new_group.suspected_tuples = suspected_tuples
   new_group.output_nodes = output_nodes
   new_group.failed_steps_and_tests = failed_steps_and_tests
   new_group.put()
 
 
 def _GetMatchingGroup(wf_failure_groups, blame_list, suspected_tuples):
   for group in wf_failure_groups:
     if _BlameListsIntersection(group.blame_list, blame_list):
       if suspected_tuples == group.suspected_tuples:
         return group
 
   return None
 
+
 def _GetOutputNodes(signals):
   if not signals or 'compile' not in signals:
     return []
 
   # Compile failures with no output nodes will be considered unique.
   return signals['compile'].get('failed_output_nodes', [])
 
 
 def _GetMatchingCompileFailureGroups(output_nodes):
   # Output nodes should already be unique and sorted.
   return WfFailureGroup.query(ndb.AND(
       WfFailureGroup.build_failure_type == failure_type.COMPILE,
       WfFailureGroup.output_nodes == output_nodes
   )).fetch()
 
 
 def _GetMatchingTestFailureGroups(failed_steps_and_tests):

[chanli, 2016/08/01 19:43:49]

Have you done anything to make sure the order for the steps and tests?

   return WfFailureGroup.query(ndb.AND(
       WfFailureGroup.build_failure_type == failure_type.TEST,
       WfFailureGroup.failed_steps_and_tests == failed_steps_and_tests
   )).fetch()
 
 
 def _IsBuildFailureUniqueAcrossPlatforms(
     master_name, builder_name, build_number, build_failure_type, blame_list,
     failed_steps, signals, heuristic_result):
   output_nodes = None
   failed_steps_and_tests = None
 
   if build_failure_type == failure_type.COMPILE:
     output_nodes = _GetOutputNodes(signals)
     if not output_nodes:
       return True
     groups = _GetMatchingCompileFailureGroups(output_nodes)
   elif build_failure_type == failure_type.TEST:
     failed_steps_and_tests = _GetStepsAndTests(failed_steps)
     if not failed_steps_and_tests:
       return True
     groups = _GetMatchingTestFailureGroups(failed_steps_and_tests)
   else:
     logging.info('Grouping %s failures is not supported. Only Compile and Test'
                  'failures can be grouped.' %
                  failure_type.GetDescriptionForFailureType(build_failure_type))
     return True
 
-  suspected_tuples = sorted(GenPotentialCulpritTupleList(heuristic_result))
+  suspected_tuples = sorted(GetSuspectedCLsWithFailures(heuristic_result))
   existing_group = _GetMatchingGroup(groups, blame_list, suspected_tuples)
 
   # Create a new WfFailureGroup if we've encountered a unique build failure.
   if existing_group:
     logging.info('A group already exists, no need for a new try job.')
     _LinkAnalysisToBuildFailureGroup(
-        master_name, builder_name, build_number, [existing_group.master_name,
-        existing_group.builder_name, existing_group.build_number])
+        master_name, builder_name, build_number,
+        [existing_group.master_name, existing_group.builder_name,
+         existing_group.build_number])
   else:
     logging.info('A new try job should be run for this unique build failure.')
     _CreateBuildFailureGroup(
         master_name, builder_name, build_number, build_failure_type, blame_list,
         suspected_tuples, output_nodes, failed_steps_and_tests)
     _LinkAnalysisToBuildFailureGroup(master_name, builder_name, build_number,
                                      [master_name, builder_name, build_number])
 
   return not existing_group
 
@@ skipping 150 matching lines @@
               pipeline.pipeline_status_path, try_job_type)
     else:  # pragma: no cover
       logging_str = (
           'Try job was scheduled for build %s, %s, %s: %s because of %s '
           'failure.') % (
               master_name, builder_name, build_number,
               pipeline.pipeline_status_path, try_job_type)
     logging.info(logging_str)
 
   return failure_result_map