Docserver overhaul: Gitiles away from me.

chrome/common/extensions/docs/server2/caching_file_system.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import posixpath
 import sys
+import traceback

[Ken Rockot(use gerrit already), 2015/05/26 00:26:23]

unused - removed in ps#2

 
 from file_system import FileSystem, StatInfo, FileNotFoundError
 from future import All, Future
 from path_util import AssertIsDirectory, IsDirectory, ToDirectory
 from third_party.json_schema_compiler.memoize import memoize
 
 
 class CachingFileSystem(FileSystem):
   '''FileSystem which implements a caching layer on top of |file_system|. If
   |fail_on_miss| is True then cache misses throw a FileNotFoundError rather than
   falling back onto the underlying FileSystem.
 
   If the underlying FileSystem is versioned (i.e., it implements GetVersion to
   return something other than None), this will create a persistent stat cache
   (keyed on the FileSystem instance's version) as an additional optimization.
   '''
   def __init__(self, file_system, object_store_creator, fail_on_miss=False):
     self._file_system = file_system
     self._fail_on_miss = fail_on_miss
-    def create_object_store(category, try_versioning=False, **optargs):
-      version = file_system.GetVersion()
-      versioned = try_versioning and version is not None
-      if versioned:
-        identity = '%s/%s' % (file_system.GetIdentity(), version)
-      else:
-        identity = file_system.GetIdentity()
-      optargs['start_empty'] = optargs.get('start_empty', not versioned)
+    def create_object_store(category, start_empty=True):
       return object_store_creator.Create(
           CachingFileSystem,
-          category='%s/%s' % (identity, category),
-          **optargs)
-    self._stat_cache = create_object_store('stat', try_versioning=True)
-    # The read caches can start populated (start_empty=False) because file
-    # updates are picked up by the stat, so it doesn't need the force-refresh
-    # which starting empty is designed for. Without this optimisation, cron
-    # runs are extra slow.
+          category='%s/%s' % (file_system.GetIdentity(), category),
+          start_empty=start_empty)
+    # We only start the stat cache empty if |fail_on_miss| is False, i.e. if
+    # we're NOT running on a live instance and we can afford to fall back onto
+    # the underlying FileSystem impl.
+    self._stat_cache = create_object_store('stat', start_empty=not fail_on_miss)
     self._read_cache = create_object_store('read', start_empty=False)
     self._walk_cache = create_object_store('walk', start_empty=False)
 
   def Refresh(self):
     return self._file_system.Refresh()
 
   def StatAsync(self, path):
     '''Stats the directory given, or if a file is given, stats the file's parent
     directory to get info about the file.
     '''
@@ skipping 16 matching lines @@
       return StatInfo(file_version)
 
     def raise_cache_miss(path):
       raise FileNotFoundError('Got cache miss when trying to stat %s' % path)
 
     dir_stat = self._stat_cache.Get(dir_path).Get()
     if dir_stat is not None:
       return Future(callback=lambda: make_stat_info(dir_stat))
 
     if self._fail_on_miss:
-      logging.warning('Bailing on stat cache miss for %s' % dir_path)
+      logging.warning('Bailing on stat cache miss for %s on %s' %
+                      (dir_path, self.GetIdentity()))
       return Future(callback=lambda: raise_cache_miss(dir_path))
 
     def next(dir_stat):
       assert dir_stat is not None  # should have raised a FileNotFoundError
       # We only ever need to cache the dir stat.
       self._stat_cache.Set(dir_path, dir_stat)
       return make_stat_info(dir_stat)
     return self._MemoizedStatAsyncFromFileSystem(dir_path).Then(next)
 
   @memoize
@@ skipping 43 matching lines @@
         (path, data) for path, (data, version) in cached_read_values.iteritems()
         if version is not None and stat_futures[path].Get().version == version)
 
     if skip_not_found:
       # Filter out paths which we know do not exist, i.e. if |path| is in
       # |cached_read_values| *and* has a None version, then it doesn't exist.
       # See the above declaration of |cached_read_values| for more information.
       paths = [path for path in paths
                if cached_read_values.get(path, (None, True))[1]]
 
-    if len(up_to_date_data) == len(paths):
+    remaining_paths = set(paths) - set(up_to_date_data.iterkeys())
+    if len(remaining_paths) == 0:
       # Everything was cached and up-to-date.
       return Future(value=up_to_date_data)
 
+    def raise_cache_miss(paths):
+      raise FileNotFoundError('Got cache miss when trying to stat %s' % paths)
+
+    if self._fail_on_miss:
+      # Ignore missing values and return anyway.
+      logging.warn('Read cache miss for %s on %s' %

[Ken Rockot(use gerrit already), 2015/05/26 00:26:23]

This was pretty useful in debugging, so I was going to leave it here.

+                   (remaining_paths, self.GetIdentity()))
+      return Future(callback=lambda: raise_cache_miss(remaining_paths))
+
     def next(new_results):
       # Update the cache. This is a path -> (data, version) mapping.
       self._read_cache.SetMulti(
           dict((path, (new_result, stat_futures[path].Get().version))
                for path, new_result in new_results.iteritems()))
       # Update the read cache to include files that weren't found, to prevent
       # constantly trying to read a file we now know doesn't exist.
       self._read_cache.SetMulti(
           dict((path, (None, None)) for path in paths
                if stat_futures[path].Get() is None))
       new_results.update(up_to_date_data)
       return new_results
+
     # Read in the values that were uncached or old.
-    return self._file_system.Read(set(paths) - set(up_to_date_data.iterkeys()),
+    return self._file_system.Read(remaining_paths,
                                   skip_not_found=skip_not_found).Then(next)
 
   def GetCommitID(self):
     return self._file_system.GetCommitID()
 
   def GetPreviousCommitID(self):
     return self._file_system.GetPreviousCommitID()
 
   def Walk(self, root, depth=-1):
     '''Overrides FileSystem.Walk() to provide caching functionality.
@@ skipping 18 matching lines @@
     return self._file_system.Walk(root, depth=depth, file_lister=file_lister)
 
   def GetIdentity(self):
     return self._file_system.GetIdentity()
 
   def GetVersion(self):
     return self._file_system.GetVersion()
 
   def __repr__(self):
     return '%s of <%s>' % (type(self).__name__, repr(self._file_system))