[Predator] Run predator.

appengine/findit/util_scripts/crash_queries/delta_test/run-predator.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import functools
+import logging
+import json
+import os
+import sys
+import threading
+import traceback
+
+_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), os.path.pardir,
+                           os.path.pardir)
+sys.path.insert(1, _SCRIPT_DIR)
+
+import script_util
+script_util.SetUpSystemPaths()
+
+from common.chrome_dependency_fetcher import ChromeDependencyFetcher
+from crash.crash_pipeline import FinditForClientID
+from crash.crash_report import CrashReport
+from crash.culprit import Culprit
+from crash.detect_regression_range import DetectRegressionRange
+from crash.findit_for_chromecrash import FinditForChromeCrash
+from crash.type_enums import CrashClient
+from crash_queries.delta_test import delta_util
+from model.crash.crash_config import CrashConfig
+from git_checkout.local_git_repository import LocalGitRepository
+import iterator
+import remote_api
+
+_FINDIT_FRACAS_FEEDBACK_URL_TEMPLATE = (
+    'https://%s.appspot.com/crash/fracas-result-feedback?key=%s')
+
+
+def GetCrashConfigFromRemoteAPI(cls, app_id, **_):
+  """Gets current crash config from datastore.
+
+  The first parameter is to take the CrashConfig instance and bound method.
+  """
+  remote_api.EnableRemoteApi(app_id)
+  # TODO(katesonia): Enable indexing by updated_ts and get the latest config.
+  version = cls._GetRootModel().query().get().current
+  for config in iterator.Iterate(cls.query(), app_id):
+    if config.key.id() == version:
+      return config
+
+  return None
+
+
+def StoreResults(crash, findit, app_id, id_to_culprits, lock, verbose=False):
+  """Stores findit result of crash into id_to_culprits dict."""
+  try:
+    stacktrace = findit._stacktrace_parser.Parse(
+        crash['stack_trace'],
+        ChromeDependencyFetcher(
+            findit._repository).GetDependency(
+                crash['crashed_version'],
+                crash['platform']))
+    if stacktrace:
+      culprit = findit._predator.FindCulprit(CrashReport(
+          crashed_version=crash['crashed_version'],
+          signature=crash['signature'],
+          platform=crash['platform'],
+          stacktrace=stacktrace,
+          regression_range=crash['regression_range']))
+    else:
+      culprit = None
+    with lock:
+      id_to_culprits[crash['id']] = culprit
+      if verbose:
+        print '\n\nCrash:', _FINDIT_FRACAS_FEEDBACK_URL_TEMPLATE % (app_id,
+                                                                    crash['id'])
+        print json.dumps(culprit.ToDicts()[0] if culprit else {'found': False},
+                         indent=4, sort_keys=True)
+  except Exception:
+    id_to_culprits[crash['id']] = None
+    print ('Expection in crash:',
+           _FINDIT_FRACAS_FEEDBACK_URL_TEMPLATE % (app_id, crash['id']))
+    raise
+
+
+def GetCulprits(crashes, client_id, app_id, verbose=False):
+  """Run predator analysis on crashes locally."""
+  origin_get = CrashConfig.Get
+  try:
+    # Mock class method Get in CrashConfig.
+    CrashConfig.Get = functools.partial(GetCrashConfigFromRemoteAPI,
+                                        CrashConfig, app_id)
+
+    findit = FinditForClientID(client_id, LocalGitRepository())
+    id_to_culprits = {}
+    lock = threading.Lock()
+    tasks = []
+    for crash in crashes:
+      crash['regression_range'] = DetectRegressionRange(
+          crash['historical_metadata'])
+      tasks.append({
+          'function': StoreResults,
+          'args': [crash, findit, app_id, id_to_culprits, lock],
+          'kwargs': {'verbose': verbose}
+      })
+    script_util.RunTasks(tasks)
+    return id_to_culprits
+  finally:
+    CrashConfig.Get = origin_get
+
+
+def RunPredator():
+  """Runs delta testing between 2 different Findit versions."""
+  argparser = argparse.ArgumentParser(
+      description='Run Predator on a batch of crashes.')
+  argparser.add_argument('result_path', help='Path to store results')
+  argparser.add_argument('client', help=('Possible values are: fracas, cracas, '
+                                         'clusterfuzz. Right now, only fracas '
+                                         'is supported.'))
+  argparser.add_argument('app', help='App engine id to get config from.')
+  argparser.add_argument(
+      '--verbose',
+      '-v',
+      action='store_true',
+      default=False,
+      help='Print findit results.')
+  args = argparser.parse_args()
+
+  crashes = json.loads(raw_input())
+  if not crashes:
+    logging.error('Failed to get crashes info.')
+    return
+
+  culprits = GetCulprits(crashes, args.client, args.app, args.verbose)
+  delta_util.FlushResult(culprits, args.result_path)
+  culprit_json = []
+  for culprit in culprits.itervalues():
+    if culprit:
+      culprit_json.append(culprit.ToDicts()[0])
+    else:
+      culprit_json.append(None)
+
+  delta_util.FlushResult(culprit_json,
+                         '%s.json' % args.result_path,
+                         serializer=json)
+
+
+if __name__ == '__main__':
+  # Disable the trivial loggings inside predator.
+  logging.basicConfig(level=logging.ERROR)
+  RunPredator()