[Predator] Run predator.

appengine/findit/util_scripts/crash_queries/delta_test/delta_test.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import json
 import logging
 import os
 import pickle
 import subprocess
 
 from crash_queries import crash_iterator
 from crash_queries.delta_test import delta_util
 
-AZALEA_RESULTS_DIRECTORY = os.path.join(os.path.dirname(__file__),
-                                        'azalea_results')
+PREDATOR_RESULTS_DIRECTORY = os.path.join(os.path.dirname(__file__),
+                                        'predator_results')
 DELTA_TEST_DIRECTORY = os.path.dirname(__file__)
+CRASH_FIELDS = ['crashed_version', 'stack_trace', 'signature',
+                'platform', 'client_id', 'regression_range',
+                'customized_data', 'historical_metadata']
 
 
 # TODO(crbug.com/662540): Add unittests.
 class Delta(object):  # pragma: no cover.
   """Stands for delta between two results.
 
   Note, the 2 results should be the same kind and have the same structure.
   """
 
   def __init__(self, result1, result2, fields):
@@ skipping 64 matching lines @@
   """
   deltas = {}
   for result_id, result1 in set1.iteritems():
     # Even when the command are exactly the same, it's possible that one set is
     # loaded from local result file, another is just queried from database,
     # sometimes some crash results would get deleted.
     if result_id not in set2:
       continue
 
     result2 = set2[result_id]
-    delta = Delta(result1, result2, result1.fields)
+    if result1:
+      fields = result1.fields
+    elif result2:
+      fields = result2.fields
+    else:
+      fields = None
+
+    delta = Delta(result1, result2, fields)
     if delta:
       deltas[result_id] = delta
 
   return deltas
 
 
 # TODO(crbug.com/662540): Add unittests.
-def GetResults(crashes, client_id, git_hash, result_path,
+def GetResults(crashes, client_id, app_id, git_hash, result_path,
                verbose=False):  # pragma: no cover.
   """Returns an evaluator function to compute delta between 2 findit githashes.
 
   Args:
     crashes (list): A list of crash infos.
     client_id (str): Possible values - fracas/cracas/clustefuzz.
+    app_id (str): Appengine app id to query.
     git_hash (str): A git hash of findit repository.
     result_path (str): file path for subprocess to write results on.
     verbose (bool): If True, print all the findit results.
 
   Return:
     A dict mapping crash id to culprit for every crashes analyzed by
     git_hash version.
   """
   if not crashes:
     return {}
 
   if verbose:
-    logging.info('\n\n***************************')
+    logging.info('***************************')
     logging.info('Switching to git %s', git_hash)
     logging.info('***************************\n\n')
 
   with open(os.devnull, 'w') as null_handle:
     subprocess.check_call(
         'cd %s; git checkout %s' % (DELTA_TEST_DIRECTORY, git_hash),
         stdout=null_handle,
         stderr=null_handle,
         shell=True)
 
   if not os.path.exists(result_path):
-    args = ['python', 'run-predator.py', result_path, '--client', client_id]
+    args = ['python', 'run-predator.py', result_path, client_id, app_id]
     if verbose:
       args.append('--verbose')
     p = subprocess.Popen(args, stdin=subprocess.PIPE)
     # TODO(katesonia): Cache crashes for crash_iterator and let subprocess read
     # corresponding cache file instead.
     p.communicate(input=json.dumps(crashes))
   else:
     logging.info('\nLoading results from %s', result_path)
 
   if not os.path.exists(result_path):
@@ skipping 32 matching lines @@
     deltas (dict): Mappings id to delta for each culprit value.
     crash_count (int): Total count of all the crashes.
   """
   head_branch_name = subprocess.check_output(
       ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).replace('\n', '')
   try:
     deltas = {}
     crash_count = 0
     for index, crashes in enumerate(
         crash_iterator.IterateCrashes(client_id, app_id,
+                                      fields=CRASH_FIELDS,
                                       property_values=property_values,
                                       start_date=start_date,
                                       end_date=end_date,
                                       batch_size=batch_size,
                                       batch_run=True)):
 
       results = []
       for git_hash in [git_hash1, git_hash2]:
         result_path = os.path.join(
-            AZALEA_RESULTS_DIRECTORY, delta_util.GenerateFileName(
+            PREDATOR_RESULTS_DIRECTORY, delta_util.GenerateFileName(
                 client_id, property_values, start_date, end_date,
                 batch_size, index, git_hash))
-        results.append(GetResults(crashes, client_id, git_hash, result_path,
+        results.append(GetResults(crashes, client_id, app_id,
+                                  git_hash, result_path,
                                   verbose=verbose))
 
       crash_count += len(crashes)
-      deltas.update(GetDeltasFromTwoSetsOfResults(*results))
-
+      batch_deltas = GetDeltasFromTwoSetsOfResults(*results)
+      # Print deltas of the current batch.
+      delta_util.PrintDelta(batch_deltas, len(crashes))
+      deltas.update(batch_deltas)
     return deltas, crash_count
   finally:
     with open(os.devnull, 'w') as null_handle:
       subprocess.check_call(['git', 'checkout', head_branch_name],
                             stdout=null_handle,
                             stderr=null_handle)