[Culprit-Finder] Seperate gae related part in cache_decorator and gitile repository to gae_libs/

appengine/findit/lib/cache_decorator.py

Index: appengine/findit/lib/cache_decorator.py
diff --git a/appengine/findit/lib/cache_decorator.py b/appengine/findit/lib/cache_decorator.py
index 07ffcb5e0af8a364b6261fd666ec313e07428b1d..993e92259a4909cfcac041cca8ae6c57dbf9fdf5 100644
--- a/appengine/findit/lib/cache_decorator.py
+++ b/appengine/findit/lib/cache_decorator.py
@@ -2,8 +2,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# TODO(http://crbug.com/660466): We should try to break dependencies.
-
 """This module provides a decorator to cache the results of a function.
 
   Examples:
@@ -36,104 +34,11 @@
       d2.Download('path')  # Returned the cached downloaded data.
 """
 
-import cStringIO
 import functools
 import hashlib
-import inspect
 import logging
+import inspect
 import pickle
-import zlib
-
-from google.appengine.api import memcache
-
-
-# TODO(katesonia): Change this to a better name, e.g. Cache.
-class Cacher(object):
-  """An interface to cache and retrieve data.
-
-  Subclasses should implement the Get/Set functions.
-  TODO: Add a Delete function (default to no-op) if needed later.
-  """
-  def Get(self, key):
-    """Returns the cached data for the given key if available.
-
-    Args:
-      key (str): The key to identify the cached data.
-    """
-    raise NotImplementedError()
-
-  def Set(self, key, data, expire_time=0):
-    """Cache the given data which is identified by the given key.
-
-    Args:
-      key (str): The key to identify the cached data.
-      data (object): The python object to be cached.
-      expire_time (int): Number of seconds from current time (up to 1 month).
-    """
-    raise NotImplementedError()
-
-
-class PickledMemCacher(Cacher):
-  """A memcache-backed implementation of the interface Cacher.
-
-  The data to be cached should be pickleable.
-  Limitation: size of the pickled data and key should be <= 1MB.
-  """
-  def Get(self, key):
-    return memcache.get(key)
-
-  def Set(self, key, data, expire_time=0):
-    return memcache.set(key, data, time=expire_time)
-
-
-class _CachedItemMetaData(object):
-  def __init__(self, number):
-    self.number = number
-
-
-class CompressedMemCacher(Cacher):
-  """A memcache-backed implementation of the interface Cacher with compression.
-
-  The data to be cached would be pickled and then compressed.
-  Data still > 1MB will be split into sub-piece and stored separately.
-  During retrieval, if any sub-piece is missing, None is returned.
-  """
-  CHUNK_SIZE = 990000
-
-  def Get(self, key):
-    data = memcache.get(key)
-    if isinstance(data, _CachedItemMetaData):
-      num = data.number
-      sub_keys = ['%s-%s' % (key, i) for i in range(num)]
-      all_data = memcache.get_multi(sub_keys)
-      if len(all_data) != num:  # Some data is missing.
-        return None
-
-      data_output = cStringIO.StringIO()
-      for sub_key in sub_keys:
-        data_output.write(all_data[sub_key])
-      data = data_output.getvalue()
-
-    return None if data is None else pickle.loads(zlib.decompress(data))
-
-  def Set(self, key, data, expire_time=0):
-    pickled_data = pickle.dumps(data)
-    compressed_data = zlib.compress(pickled_data)
-
-    all_data = {}
-    if len(compressed_data) > self.CHUNK_SIZE:
-      num = 0
-      for index in range(0, len(compressed_data), self.CHUNK_SIZE):
-        sub_key = '%s-%s' % (key, num)
-        all_data[sub_key] = compressed_data[index : index + self.CHUNK_SIZE]
-        num += 1
-
-      all_data[key] = _CachedItemMetaData(num)
-    else:
-      all_data[key] = compressed_data
-
-    keys_not_set = memcache.set_multi(all_data, time=expire_time)
-    return len(keys_not_set) == 0
 
 
 def _DefaultKeyGenerator(func, args, kwargs):
@@ -161,9 +66,9 @@ def _DefaultKeyGenerator(func, args, kwargs):
 
 
 def Cached(namespace=None,
-           expire_time=0,
+           expire_time=-1,
            key_generator=_DefaultKeyGenerator,
-           cacher=PickledMemCacher()):
+           cache=None):
   """Returns a decorator to cache the decorated function's results.
 
   However, if the function returns None, empty list/dict, empty string, or other
@@ -178,18 +83,18 @@ def Cached(namespace=None,
   - If the default key generator is used, parameters passed to the decorated
     function should be pickleable, or each of the parameter has an identifier
     property or method which returns pickleable results.
-  - If the default cacher is used, the returned results of the decorated
+  - If the default cache is used, the returned results of the decorated
     function should be pickleable.
 
   Args:
     namespace (str): A prefix to the key for the cache. Default to the
         combination of module name and function name of the decorated function.
     expire_time (int): Expiration time, relative number of seconds from current
-        time (up to 1 month). Defaults to 0 -- never expire.
+        time (up to 0 month). Defaults to 0 -- never expire.

[wrengr, 2016/12/06 21:52:57]

it now defaults to -1 rather than 0...

[Sharu Jiang, 2016/12/06 23:58:17]

After rebase this should defaults to 0, old version...

     key_generator (function): A function to generate a key to represent a call
         to the decorated function. Defaults to :func:`_DefaultKeyGenerator`.
-    cacher (Cacher): An instance of an implementation of interface `Cacher`.
-        Defaults to one of `PickledMemCacher` which is based on memcache.
+    cache (Cache): An instance of an implementation of interface `Cache`.
+        Defaults to None.
 
   Returns:
     The cached results or the results of a new run of the decorated function.
@@ -205,7 +110,7 @@ def Cached(namespace=None,
       key = '%s-%s' % (prefix, key_generator(func, args, kwargs))
 
       try:
-        result = cacher.Get(key)
+        result = cache.Get(key)
       except Exception:  # pragma: no cover.
         result = None
         logging.exception(
@@ -218,7 +123,7 @@ def Cached(namespace=None,
       result = func(*args, **kwargs)
       if result:
         try:
-          cacher.Set(key, result, expire_time=expire_time)
+          cache.Set(key, result, expire_time=expire_time)
         except Exception:  # pragma: no cover.
           logging.exception(
               'Failed to cache data for function %s.%s, args=%s, kwargs=%s',