[Findit] Add iterator and crash_iterator for delta test

appengine/findit/util_scripts/crash_queries/crash_iterator.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import base64
+import copy
+from datetime import date
+from datetime import datetime
+from datetime import timedelta
+import logging
+import json
+import os
+import zlib

[stgao, 2016/10/10 23:39:39]

Need clean-up.

[Sharu Jiang, 2016/10/12 00:52:10]

Done.

+
+import dev_appserver
+dev_appserver.fix_sys_path()
+
+import iterator
+from crash import findit_for_client
+from crash.type_enums import CrashClient
+from model.crash.fracas_crash_analysis import FracasCrashAnalysis
+from model.crash.cracas_crash_analysis import CracasCrashAnalysis
+import remote_api
+
+_DEFAULT_BATCH_SIZE = 1000
+_TODAY = date.today().strftime('%Y-%m-%d')
+_A_YEAR_AGO = (date.today() - timedelta(days=365)).strftime('%Y-%m-%d')
+
+COMMON_CRASH_FIELDS = ['crashed_version', 'stack_trace', 'signature',
+                       'platform', 'client_id', 'customized_data']
+
+
+# TODO(katesonia): Switch to use fuction of objects encapsulating CrashClients,
+# after the refactoring is done.
+def GetAnalysisClassForClient(client_id):
+  if client_id == CrashClient.FRACAS:
+    return FracasCrashAnalysis
+  elif client_id == CrashClient.CRACAS:
+    return CracasCrashAnalysis
+  elif client_id == CrashClient.CLUSTERFUZZ:
+    # TODO(katesonia): Define ClusterfuzzCrashAnalysis.
+    return None
+
+  return None
+
+
+def GetQueryForClient(client_id, property_values, start_date, end_date,
+                      datetime_pattern='%Y-%m-%d'):
+  if property_values is None:
+    property_values = {}
+
+  start_date = datetime.strptime(start_date, datetime_pattern)
+  end_date = datetime.strptime(end_date, datetime_pattern)
+  cls = GetAnalysisClassForClient(client_id)
+  return cls.query(**property_values).filter(

[stgao, 2016/10/10 23:39:39]

Can we use https://cloud.google.com/appengine/docs/python/ndb/projectionqueries
so that those fields don't have to be passed over into the iterator?
This could make the interface cleaner.

[Sharu Jiang, 2016/10/12 00:52:10]

Problem is only indexed property can be projected, for some properties like
stacktrace, we don't want to index them.

+      cls.requested_time >= start_date).filter(
+      cls.requested_time < end_date)
+
+
+def IterateCrashes(client_id,
+                   fields=None,
+                   property_values=None,
+                   start_date=_A_YEAR_AGO,
+                   end_date=_TODAY,
+                   batch_size=_DEFAULT_BATCH_SIZE,
+                   batch_run=False):
+  """Genrates query to query crashes and iterates crashes.
+
+  Args:
+    client_id (CrashClient): One of CrashClient.FRACAS, CrashClient.CRACAS,
+      CrashClient.CLUSTERFUZZ.
+    fields (list): Field names of CrashAnalysis entity to project.
+    property_values (dict): Property values to query.

[stgao, 2016/10/10 23:39:39]

nit: "to filter"?

[Sharu Jiang, 2016/10/12 00:52:10]

Done.

+    start_date (str): Only iterate testcases after this date including this
+      date, format '%Y-%m-%d'.
+    end_date (str): Only iterate testcases before this date excluding this date,
+      format '%Y-%m-%d'.
+    batch_size (int): The number of crashes to query at one time.
+    batch_run (bool): If True, iterate batches of crashes, if
+      False, iterate each crash.
+
+    An example is available in crash_printer/print_crash.py.
+  """
+  if fields is None:
+    fields = COMMON_CRASH_FIELDS
+
+  if property_values is None:
+    property_values = {}
+
+  query = GetQueryForClient(client_id, property_values, start_date, end_date)
+  for crash in iterator.Iterate(query, fields, batch_size=batch_size,
+                                batch_run=batch_run):
+    yield crash