[Predator] Add local cache for get command output.

appengine/findit/common/cache_decorator.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This module provides a decorator to cache the results of a function.
 
   Examples:
   1. Decorate a function:
     @cache_decorator.Cached()
     def Test(a):
@@ skipping 21 matching lines @@
 
       d2 = Downloader('http://url', 5)
       d2.Download('path')  # Returned the cached downloaded data.
 """
 
 import cStringIO
 import functools
 import hashlib
 import inspect
 import logging
+import os
 import pickle
+import threading
 import zlib
 
 from google.appengine.api import memcache
 
+_DEFAULT_LOCAL_CACHE_DIR = os.path.join(os.path.expanduser('~'),
+                                        '.culprit_finder', 'local_cache')
+
 
 class Cacher(object):
   """An interface to cache and retrieve data.
 
   Subclasses should implement the Get/Set functions.
   TODO: Add a Delete function (default to no-op) if needed later.
   """
   def Get(self, key):
     """Returns the cached data for the given key if available.
 
@@ skipping 69 matching lines @@
         num += 1
 
       all_data[key] = _CachedItemMetaData(num)
     else:
       all_data[key] = compressed_data
 
     keys_not_set = memcache.set_multi(all_data, time=expire_time)
     return len(keys_not_set) == 0
 
 
+class LocalCacher(Cacher):
+  """Cacher that uses local files to cache data."""
+
+  lock = threading.Lock()
+
+  def __init__(self, cache_dir=_DEFAULT_LOCAL_CACHE_DIR):

[stgao, 2016/11/02 02:15:07]

Can we avoid the default value, and let client pass it over instead?

[Sharu Jiang, 2016/11/08 01:17:12]

Done.

+    self.cache_dir = cache_dir
+    if not os.path.exists(cache_dir):  # pragma: no cover.
+      os.makedirs(cache_dir)
+
+  def Get(self, key):
+    with LocalCacher.lock:
+      path = os.path.join(self.cache_dir, key)
+      if not os.path.exists(path):
+        return None
+
+      try:
+        with open(path) as f:
+          return pickle.loads(zlib.decompress(f.read()))
+      except Exception as e:  # pragma: no cover.
+        logging.error('Failed loading cache: %s', e)
+        return None
+
+  def Set(self, key, data):  # pylint: disable=W
+    with LocalCacher.lock:
+      try:
+        with open(os.path.join(self.cache_dir, key), 'wb') as f:
+          f.write(zlib.compress(pickle.dumps(data)))
+      except Exception as e:  # pragma: no cover.
+        logging.error('Failed setting cache for key %s: %s', key, e)
+
+
 def _DefaultKeyGenerator(func, args, kwargs):
   """Generates a key from the function and arguments passed to it.
 
   Args:
     func (function): An arbitrary function.
     args (list): Positional arguments passed to ``func``.
     kwargs (dict): Keyword arguments passed to ``func``.
 
   Returns:
     A string to represent a call to the given function with the given arguments.
@@ skipping 73 matching lines @@
         except Exception:  # pragma: no cover.
           logging.exception(
               'Failed to cache data for function %s.%s, args=%s, kwargs=%s',
               func.__module__, func.__name__, repr(args), repr(kwargs))
 
       return result
 
     return Wrapped
 
   return Decorator