[Findit] Add skeleton code for delta test script.

appengine/findit/util_scripts/crash_queries/delta_test/delta_test.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import logging
+import os
+import pickle
+import subprocess
+
+from crash_queries import crash_iterator
+from crash_queries.delta_test import delta_util
+
+AZALEA_RESULTS_DIRECTORY = os.path.join(os.path.dirname(__file__),
+                                        'azalea_results')
+
+
+class Delta(object):
+  """Stands for delta between two results."""
+
+  def __init__(self, result1, result2, fields):
+    self._result1 = result1
+    self._result2 = result2
+    self._fields = fields
+    self._delta_dict = {}
+    self._delta_str_dict = {}
+
+  @property
+  def delta_dict(self):
+    """Dict representation of delta."""
+    if self._delta_dict:
+      return self._delta_dict
+
+    for field in self._fields:
+      value1 = getattr(self._result1, field)
+      value2 = getattr(self._result2, field)
+      if value1 != value2:
+        if hasattr(value1, 'ToDict') and callable(value1.ToDict):
+          value1 = value1.ToDict()
+          value2 = value2.ToDict()
+        self._delta_dict[field] = (value1, value2)
+
+    return self._delta_dict
+
+  @property
+  def delta_str_dict(self):
+    """Converts delta of each field to a string."""
+    if self._delta_str_dict:
+      return self._delta_str_dict
+
+    for key, (value1, value2) in self.delta_dict.iteritems():
+      self._delta_str_dict[key] = '%s: %s, %s' % (key, value1, value2)
+
+    return self._delta_str_dict
+
+  def ToDict(self):
+    return self.delta_dict
+
+  def __str__(self):
+    return '\n'.join(self.delta_str_dict.values())
+
+  def __bool__(self):
+    return bool(self.delta_dict)
+
+  def __nonzero__(self):
+    return self.__bool__()
+
+
+def GetDeltasFromTwoSetsOfResults(set1, set2):
+  """Gets delta from two sets of results.
+
+  Results are a list of (message, matches, component_name, cr_label)
+  Returns a list of delta results (results1, results2).
+  """
+  deltas = {}
+  for crash_id, result1 in set1.iteritems():
+    # Even when the command are exactly the same, it's possible that one set is
+    # loaded from local result file, another is just queried from database,
+    # sometimes some crash results would get deleted.
+    if crash_id not in set2:
+      continue
+
+    result2 = set2[crash_id]
+    delta = Delta(result1, result2, result1.fields)
+    if delta:
+      deltas[crash_id] = delta
+
+  return deltas
+
+
+def GetResults(crashes, git_hash, result_path, verbose=False):
+  """Returns an evaluator function to compute delta between 2 findit githashes.
+
+  Args:
+    crashes (list): A list of crash infos.
+    git_hash (str): A git hash of findit repository.
+    result_path (str): file path for subprocess to write results on.
+    verbose (bool): If True, print all the findit results.
+
+  Return:
+    A dict mapping crash id to culprit for every crashes analyzed by
+    git_hash version.
+  """
+  if not crashes:
+    return {}
+
+  if verbose:
+    logging.info('\n\n***************************')
+    logging.info('Switch to git %s', git_hash)
+    logging.info('***************************\n\n')
+
+  dev_null_handle = open(os.devnull, 'w')
+  subprocess.check_call(
+      'cd %s; git checkout %s' % (os.path.dirname(__file__), git_hash),
+      stdout=dev_null_handle,
+      stderr=dev_null_handle,
+      shell=True)
+
+  if not os.path.exists(result_path):
+    # TODO(katesoina): Implement run-azalea.py.

[stgao, 2016/10/14 01:40:40]

Is the TODO still valid?

[Sharu Jiang, 2016/10/15 01:24:47]

run-azalea.py is not finished yet, however, right, better add Todo there instead
of here.

+    command = 'python %s %s' % ('run-azalea.py', result_path) + (

[stgao, 2016/10/14 01:40:40]

what's the current working dir? Will it matter here?

[Sharu Jiang, 2016/10/15 01:24:47]

In line 114, we make sure the current working dir is the delta test dir.

[stgao, 2016/10/20 01:40:20]

Is this new process run in the same shell as that process in #115 above? Did the
`cd DIR` really change the current working dir? Have you tried in a command
line?

[Sharu Jiang, 2016/10/20 22:39:06]

Yes, the run-azalea.py is in the same dir as this file(delta test dir). I tested
the skeleton delta test in command line, it works.

+        ' -v' if verbose else '')

[stgao, 2016/10/14 01:40:40]

Can we use a argument list instead of a string?

[Sharu Jiang, 2016/10/15 01:24:47]

Done.

+    # Results is a dict with testcase_id as key, and findit results as
+    # value.
+    p = subprocess.Popen(
+        command,
+        stdin=subprocess.PIPE,
+        shell=True)
+
+    p.communicate(input=json.dumps(crashes))

[stgao, 2016/10/14 01:40:40]

Should we pass information through a file instead?

[Sharu Jiang, 2016/10/15 01:24:47]

We can, we can cache the CrashIterator results(batches of crashes) and let
run-azalea read those cache, but as I stated in the design doc
https://docs.google.com/document/d/1kfu_HIJFCSwOWQleRpSkOH4t_D0UFY5vDSSausaZu...,
this may caused some disk space since whenever args changed, we need to cache
those crash_infos (it's relatively big compared to culprit result) again. 

I can do this in another cl.

[stgao, 2016/10/20 01:40:20]

I don't quite understand this. Maybe we could chat in person.
If this is the plan, let's add a TODO with a bug.

[Sharu Jiang, 2016/10/20 22:39:06]

Done.

+  else:
+    logging.info('\nLoading results from %s', result_path)
+
+  if not os.path.exists(result_path):
+    logging.info('Fail to get results.')

[stgao, 2016/10/14 01:40:40]

Should it be an error or info? Same for other usage of logging.

[Sharu Jiang, 2016/10/15 01:24:47]

Done.

+    return {}
+
+  with open(result_path) as f:
+    return pickle.load(f)

[stgao, 2016/10/14 01:40:40]

Just curious: why we use pickle instead of json?

[Sharu Jiang, 2016/10/15 01:24:47]

The results are a general concept, it can be a object like Culprit object.

[stgao, 2016/10/20 01:40:20]

Acknowledged.

+
+  return {}
+
+
+def DeltaEvaluator(git_hash1, git_hash2,
+                   client_id, start_date, end_date, batch_size,
+                   property_values=None, verbose=False, app_id=None):
+  """Evaluates delta between git_hash1 and git_hash2 on a set of Testcases.
+
+  Args:
+    git_hash1 (str): A git hash of findit repository.
+    git_hash2 (str): A git hash of findit repository.
+    start_date (str): Run delta test on testcases after (including)
+      the start_date, format should be '%Y-%m-%d'.
+    end_date (str): Run delta test on testcases before (not including)
+      the end_date, format should be '%Y-%m-%d'.
+    client_id (CrashClient): Possible values are 'fracas', 'cracas',
+      'cluterfuzz'.
+    batch_size (int): Size of a batch that can be queried at one time.
+    property_values (dict): Property values to query.
+       batch_size (int): The size of crashes that can be queried at one time.
+    verbose (bool): If True, print all the findit results.
+    app_id (str): Appengine app id to query.
+  Return:
+    (deltas, crash_count).
+    deltas (dict): Mappings id to delta for each culprit value.
+    crash_count (int): Total count of all the crashes.
+  """
+  head_branch_name = subprocess.check_output(
+      ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).replace('\n', '')
+  deltas = {}
+  try:
+    crash_count = 0
+    for index, crashes in enumerate(
+        crash_iterator.IterateCrashes(client_id,
+                                      property_values=property_values,
+                                      start_date=start_date,
+                                      end_date=end_date,
+                                      batch_size=batch_size,
+                                      batch_run=True,
+                                      app_id=app_id)):
+
+      results = []
+      for git_hash in [git_hash1, git_hash2]:

[stgao, 2016/10/14 01:40:40]

So for each crash, we switch the checkout twice. If we have 100 crashes, we have
to switch checkout 200 times.

Can we avoid such an overhead?

[Sharu Jiang, 2016/10/15 01:24:47]

This is not for each crash, it is for each batch of 1000 crashes
(batch_run=True). So I think the overhead should be ok.

[stgao, 2016/10/20 01:40:20]

sg

[wrengr, 2016/10/24 17:30:36]

It'd still be good (and easy!) to avoid. I can add it in another CL once this
one finally lands.

[Sharu Jiang, 2016/10/24 19:14:32]

Can we discuss the solution offline then?

The main reasons I want to get delta result batch by batch are
1) we can print intermediate delta result to users (will print that in another
cl), especially when users query a long period of crash analyses, it may take
long time to see the final results.
2) we can also avoid caching a very big chunk of crashes in memory (or writing
to disks), especially when users query a long period of crash analyses.

+        result_path = os.path.join(
+            AZALEA_RESULTS_DIRECTORY, delta_util.GenerateResultFileName(
+                client_id, property_values, start_date, end_date,
+                batch_size, index, git_hash))
+        results.append(GetResults(crashes, git_hash, result_path,
+                                  verbose=verbose))
+
+      crash_count += len(crashes)
+      deltas.update(GetDeltasFromTwoSetsOfResults(*results))
+
+    return deltas, crash_count
+  finally:
+    dev_null_handle = open(os.devnull, 'w')
+    subprocess.check_call(['git', 'checkout', head_branch_name],
+                          stdout=dev_null_handle,
+                          stderr=dev_null_handle)