Extensions Docs Server: Efficient MemcacheFileSystem

chrome/common/extensions/docs/server2/memcache_file_system.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from file_system import FileSystem
 from future import Future
 import appengine_memcache as memcache
 
 class MemcacheFileSystem(FileSystem):
   """FileSystem implementation which memcaches the results of Read.
   """
   def __init__(self, file_system, memcache):
     self._file_system = file_system
     self._memcache = memcache
 
   def Stat(self, path):
     """Stats the directory given, or if a file is given, the file's parent
     directory.
     """
     version = self._memcache.Get(path, memcache.MEMCACHE_FILE_SYSTEM_STAT)
     if version is None:
       stat_info = self._file_system.Stat(path)
       self._memcache.Set(path,
                          stat_info.version,
                          memcache.MEMCACHE_FILE_SYSTEM_STAT)
+      for child_path, child_version in stat_info.child_versions.iteritems():
+        self._memcache.Set(path.rsplit('/', 1)[0] + '/' + child_path,
+                           child_version,
+                           memcache.MEMCACHE_FILE_SYSTEM_STAT)
     else:
-      stat_info = self.StatInfo(version)
+      stat_info = self.StatInfo(version, {})

[not at google - send to devlin, 2012/08/13 05:34:15]

memcache can store arbitrary objects, I believe. If you store the StatInfo
object in memcache rather than just the version then you won't need to construct
this fake (and incorrect) {} object.

[cduvall, 2012/08/13 19:45:45]

Memcache can store dictionaries, but not objects. They need to have a pickle
method I believe. I changed this to use None instead of {}, which I think is a
little less confusing.

[not at google - send to devlin, 2012/08/13 23:02:14]

I just checked and it looks like pickle can handle at least simple objects (I
tested objects with simple and list fields). So... let's at least try it.

Here is my output from the python console, by the way:
Python 2.7.3 (default, Apr 13 2012, 00:05:08) 
[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
...   def __init__(self, x, xs):
...     self.x = x
...     self.xs = xs
... 
False
10
[1, 2, 3]

[cduvall, 2012/08/14 18:15:00]

Aha! The problem with pickle was nested classes. It can't pickle a nested class.
Here's my output:
"PicklingError: Can't pickle <class 'file_system.StatInfo'>: attribute lookup
file_system.StatInfo failed"

I'm going to leave this just storing the version for now, and I can change it
later if we want.

[not at google - send to devlin, 2012/08/15 05:24:05]

Ok, let's un-nest StatInfo then (or figure out how to make pickle work).

     return stat_info
 
   def Read(self, paths, binary=False):
     """Reads a list of files. If a file is in memcache and it is not out of
     date, it is returned. Otherwise, the file is retrieved from the file system.
     """
     result = {}
     uncached = []
     for path in paths:
       cached_result = self._memcache.Get(path,
                                          memcache.MEMCACHE_FILE_SYSTEM_READ)
       if cached_result is None:
         uncached.append(path)
         continue
       data, version = cached_result
       if self.Stat(path).version > version:
         self._memcache.Delete(path, memcache.MEMCACHE_FILE_SYSTEM_READ)
         uncached.append(path)
         continue
       result[path] = data
     new_items = self._file_system.Read(uncached, binary=binary).Get()
     for item in new_items:
       version = self.Stat(item).version
       value = new_items[item]
       self._memcache.Set(item,
                          (value, version),
                          memcache.MEMCACHE_FILE_SYSTEM_READ)
       result[item] = value
     return Future(value=result)