[Findit] Add iterator and crash_iterator for delta test

appengine/findit/util_scripts/crash_queries/crash_iterator.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Fetches crashes and iterate over and process them."""
+
+import os
+
+import remote_api
+
+_DEFAULT_BATCH_SIZE = 1000
+
+
+def GetCrashInfo(crash, crash_info_fields):

[wrengr, 2016/10/05 18:41:58]

This seems very unclean. Why not just use the |crash| directly? If you want
certain properties to default to None, then use a class that does so directly.
(E.g., have a factory that will construct a class from |crash_info_fields|, and
then just pass the |crash| to the constructor for that class.)

[Sharu Jiang, 2016/10/06 18:16:56]

This is for saving memory especially for the batch_run=True, since there are
many fields in the CrashAnalysis that are not used. This crashes will be PIPE to
subprocess to run findit, better keep it small.

+  """Gets crash_info_fields from crash object. Returns dict."""
+  crash_info = {}
+  for field in crash_info_fields:
+    if hasattr(crash, field):
+      crash_info[field] = getattr(crash, field)
+    else:
+      crash_info[field] = None
+  crash_info['id'] = crash.key.id()
+  return crash_info
+
+
+def IterateCrashes(query,
+                   crash_info_fields,
+                   callback_func,

[wrengr, 2016/10/05 18:41:58]

See my note about callback functions vs generators in the previous file

[Sharu Jiang, 2016/10/06 18:16:56]

Acknowledged.

+                   filter_func=None,
+                   batch_size=_DEFAULT_BATCH_SIZE,
+                   batch_run=False):
+  """Pulls crashes and passes needed info to callback function.
+
+  Args:
+    query (ndb.Query): The query to fetch crahes.
+    crash_info_fields (list): Field names of a crashto pass over to the
+        callback function. All supported fields are in class Testcase in
+        clusterfuzz/src/common/data_types.py. If a given field name is not
+        available, it is set to None. 'testcase_id' is always added by default
+        as an integer.
+    callback_func (function): A function with a dict as the single parameter.
+        For example:
+          callback(testcase_info):
+            print 'Testcase %d has fields:' % testcase_info['testcase_id']
+            for name, value in testcase_info.iteritems():
+              print '  %s: %s' % (name, value)
+    filter_func (function): A function that filters crashes.
+    batch_size (int): The number of crashes to query at one time.
+    batch_run (bool): If True, run callback_func on a batch of crashes, if
+      False, run callback_func on each crash.
+
+    An exmaple is available in crash_printer/print_crash.py.
+  """
+  remote_api.EnableRemoteApi(app_id=os.getenv('APP_ID'))
+
+  cursor = None
+  while True:
+    crashes, next_cursor, more = query.fetch_page(batch_size,
+                                                  start_cursor=cursor)
+
+    if not more:
+      break
+
+    if filter_func:
+      crashes = filter_func(crashes)
+
+    crashes = [GetCrashInfo(crash, crash_info_fields) for crash in crashes]
+
+    if batch_run:
+      callback_func(crashes)
+    else:
+      for crash in crashes:
+        callback_func(crash)
+
+    cursor = next_cursor