[Findit] Add iterator and crash_iterator for delta test

appengine/findit/util_scripts/iterator.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Fetches entities and iterate over and process them."""
+
+import os
+
+import remote_api
+
+_DEFAULT_BATCH_SIZE = 1000
+
+
+def ProjectEntity(entity, fields):
+  """Projects fields from entity. Returns dict."""
+  entity_info = {}
+  for field in fields:
+    if hasattr(entity, field):
+      entity_info[field] = getattr(entity, field)
+    else:
+      entity_info[field] = None
+  entity_info['id'] = entity.key.id()
+  return entity_info
+
+
+def Iterate(query,
+            fields,
+            filter_func=None,
+            batch_size=_DEFAULT_BATCH_SIZE,
+            batch_run=False,
+            app_id=None):
+  """Iterates entities queried by query.
+
+  Args:
+    query (ndb.Query): The query to fetch entities.
+    fields (list): Field names of an entity to be projected to a dict.
+      If a given field name is not available, it is set to None.
+      'id' is always added by default as an integer.
+    filter_func (function): A function that does in memory filtering.
+    batch_size (int): The number of entities to query at one time.
+    batch_run (bool): If True, iterate batches of entities, if
+      False, iterate each entity.
+    app_id (str): App engine app id.
+
+    An exmaple is available in crash_printer/print_crash.py.
+  """
+  if app_id:
+    remote_api.EnableRemoteApi(app_id)
+  else:
+    remote_api.EnableRemoteApi()

[stgao, 2016/10/13 06:38:52]

Why we don'd expect an app id? What's the default one?

[Sharu Jiang, 2016/10/13 20:31:16]

I think this iterator can be used not only for crash, it can be potentially used
by waterfall, when there is no app_id specified, it just falls back to whatever
default value the remote_api has.

[stgao, 2016/10/14 00:55:01]

It could be reused, but this style of default value could make it really hard to
debug when error occurs.

[stgao, 2016/10/19 00:49:47]

Ping.

[Sharu Jiang, 2016/10/19 20:12:44]

Done.

+
+  cursor = None
+  while True:
+    entities, next_cursor, more = query.fetch_page(batch_size,
+                                                   start_cursor=cursor)
+
+    if not more:
+      break
+
+    if filter_func:
+      entities = filter_func(entities)
+
+    entities = [ProjectEntity(entity, fields) for entity in entities]
+
+    if batch_run:
+      yield entities
+    else:
+      for entity in entities:
+        yield entity
+
+    cursor = next_cursor