Docserver: Persist stat cache for versioned file systems

chrome/common/extensions/docs/server2/caching_file_system.py

Index: chrome/common/extensions/docs/server2/caching_file_system.py
diff --git a/chrome/common/extensions/docs/server2/caching_file_system.py b/chrome/common/extensions/docs/server2/caching_file_system.py
index b0573716f1fbc2fa56cc68e163bd5d627a9f2856..3673096530a381373bbaaebd27b47deda8add90c 100644
--- a/chrome/common/extensions/docs/server2/caching_file_system.py
+++ b/chrome/common/extensions/docs/server2/caching_file_system.py
@@ -11,19 +11,49 @@ from path_util import AssertIsDirectory, IsDirectory, ToDirectory
 from third_party.json_schema_compiler.memoize import memoize
 
 
+
+class CacheMissError(Exception):
+  '''Raise when data is not found in a CachingFileSystem which is not allowed
+  to reference its backing FileSystem.'''
+  def __init__(self, message):
+    Exception.__init__(self, message)
+
+
+
 class CachingFileSystem(FileSystem):
-  '''FileSystem which implements a caching layer on top of |file_system|. It's
-  smart, using Stat() to decided whether to skip Read()ing from |file_system|,
-  and only Stat()ing directories never files.
+  '''FileSystem which implements a caching layer on top of |file_system|. If
+  |fail_on_miss| is True then cache misses throw a CacheMissError rather than
+  falling back onto the underlying FileSystem.
+
+  If |empty_stat_cache| is True (default), its stat cache is initialized empty.
+  This should be set to False when wrapping a FileSystem that makes a proper
+  distinction between stable identity and unstable identity.
+
+  The working assumption is that a file system's unstable identity changes any
+  time any contents of the file system change, and therefore a stat cache keyed
+  on unstable identity will never need to be refreshed.
   '''
-  def __init__(self, file_system, object_store_creator):
+  def __init__(self,
+               file_system,
+               object_store_creator,
+               fail_on_miss=False,
+               empty_stat_cache=True):
     self._file_system = file_system
-    def create_object_store(category, **optargs):
+    self._fail_on_miss = fail_on_miss
+    def create_object_store(category, use_stable_identity=True, **optargs):
+      if use_stable_identity:
+        identity = file_system.GetStableIdentity()
+      else:
+        identity = file_system.GetUnstableIdentity()
       return object_store_creator.Create(
           CachingFileSystem,
-          category='%s/%s' % (file_system.GetIdentity(), category),
+          category='%s/%s' % (identity, category),
           **optargs)
-    self._stat_cache = create_object_store('stat')
+    # The stable stat cache caches file stat info keyed by the file system's
+    # stable identity (or unstable identity for persistent caches).
+    self._stat_cache = create_object_store('stat',
+        use_stable_identity=empty_stat_cache,
+        start_empty=empty_stat_cache)
     # The read caches can start populated (start_empty=False) because file
     # updates are picked up by the stat, so it doesn't need the force-refresh
     # which starting empty is designed for. Without this optimisation, cron
@@ -56,10 +86,17 @@ class CachingFileSystem(FileSystem):
                                 (path, dir_path, dir_stat.child_versions))
       return StatInfo(file_version)
 
+    def raise_cache_miss(path):
+      raise FileNotFoundError('Got cache miss when trying to stat %s' % path)
+
     dir_stat = self._stat_cache.Get(dir_path).Get()
     if dir_stat is not None:
       return Future(callback=lambda: make_stat_info(dir_stat))
 
+    if self._fail_on_miss:
+      logging.info('Bailing on stat cache miss for %s' % dir_path)
+      return Future(callback=lambda: raise_cache_miss(dir_path))
+
     def next(dir_stat):
       assert dir_stat is not None  # should have raised a FileNotFoundError
       # We only ever need to cache the dir stat.
@@ -84,7 +121,7 @@ class CachingFileSystem(FileSystem):
     # (path, None, None). This is to prevent re-reads of files we know
     # do not exist.
     cached_read_values = self._read_cache.GetMulti(paths).Get()
-    cached_stat_values = self._stat_cache.GetMulti(paths).Get()
+    cached_stat_info = self._stat_cache.GetMulti(paths).Get()
 
     # Populate a map of paths to Futures to their stat. They may have already
     # been cached in which case their Future will already have been constructed
@@ -97,13 +134,13 @@ class CachingFileSystem(FileSystem):
       raise error
 
     for path in paths:
-      stat_value = cached_stat_values.get(path)
-      if stat_value is None:
+      stat_info = cached_stat_info.get(path)
+      if stat_info is None:
         stat_future = self.StatAsync(path)
         if skip_not_found:
           stat_future = stat_future.Then(lambda x: x, handle)
       else:
-        stat_future = Future(value=stat_value)
+        stat_future = Future(value=stat_info)
       stat_futures[path] = stat_future
 
     # Filter only the cached data which is up to date by comparing to the latest
@@ -169,12 +206,6 @@ class CachingFileSystem(FileSystem):
       return dirs, files
     return self._file_system.Walk(root, depth=depth, file_lister=file_lister)
 
-  def GetCommitID(self):
-    return self._file_system.GetCommitID()
-
-  def GetPreviousCommitID(self):
-    return self._file_system.GetPreviousCommitID()
-
   def GetIdentity(self):
     return self._file_system.GetIdentity()