Refactor the devserver to make it easier to control caching

chrome/common/extensions/docs/server2/compiled_file_system.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import os
-
-import object_store
-
-APPS                      = 'Apps'
-APPS_FS                   = 'AppsFileSystem'
-CRON                      = 'Cron'
-EXTENSIONS                = 'Extensions'
-EXTENSIONS_FS             = 'ExtensionsFileSystem'
-CRON_FILE_LISTING         = 'Cron.FileListing'
-CRON_GITHUB_INVALIDATION  = 'Cron.GithubInvalidation'
-CRON_INVALIDATION         = 'Cron.Invalidation'
-HANDLEBAR                 = 'Handlebar'
-IDL                       = 'IDL'
-IDL_NO_REFS               = 'IDLNoRefs'
-IDL_NAMES                 = 'IDLNames'
-INTRO                     = 'Intro'
-JSON                      = 'JSON'
-JSON_NO_REFS              = 'JSONNoRefs'
-LIST                      = 'List'
-NAMES                     = 'Names'
-PERMS                     = 'Perms'
-SIDENAV                   = 'Sidenav'
-STATIC                    = 'Static'
-ZIP                       = 'Zip'
+from object_store_creator import ObjectStoreCreator
 
 class _CacheEntry(object):
   def __init__(self, cache_data, version):
     self._cache_data = cache_data
     self.version = version
 
 class CompiledFileSystem(object):
   """This class caches FileSystem data that has been processed.
   """
   class Factory(object):
-    """A class to build a CompiledFileSystem.
+    """A class to build a CompiledFileSystem backed by |file_system|.
+    Set an explicit |store_type| for tests.
     """
-    def __init__(self, file_system, object_store):
+    def __init__(self, file_system, store_type=None):
       self._file_system = file_system
-      self._object_store = object_store
+      self._store_type = store_type
+      self._identity_instance = None
 
-    def Create(self, populate_function, namespace, version=None):
+    def Create(self, populate_function, cls, category=None, version=None):
       """Create a CompiledFileSystem that populates the cache by calling
       |populate_function| with (path, data), where |data| is the data that was
-      fetched from |path|. The keys to the cache are put in the namespace
-      specified by |namespace|, and optionally adding |version|.  """
-      return CompiledFileSystem(self._file_system,
-                                populate_function,
-                                self._object_store,
-                                namespace,
-                                version=version)
+      fetched from |path|.
+      The namespace for the file system is derived like ObjectStoreCreator: from
+      |cls| along with an optional |category| and |version|.
+      """
+      assert isinstance(cls, type)
+      assert not cls.__name__[0].islower()  # guard against non-class types
+      full_name = cls.__name__
+      if category is not None:
+        full_name = '%s/%s' % (full_name, category)
+      return self._Create(populate_function, full_name, version=version)
+
+    def GetOrCreateIdentity(self):
+      '''Handy helper to get or create the identity compiled file system.
+      GetFromFile will return the file's contents.
+      GetFromFileListing will return the directory list.
+      '''
+      if self._identity_instance is None:
+        self._identity_instance = self._Create(lambda _, x: x, 'id')
+      return self._identity_instance
+
+    def _Create(self, populate_function, full_name, version=None):
+      object_store_creator = ObjectStoreCreator(CompiledFileSystem,
+                                                store_type=self._store_type)
+      return CompiledFileSystem(
+          self._file_system,
+          populate_function,
+          object_store_creator.Create(category='%s/file' % full_name,
+                                      version=version),
+          object_store_creator.Create(category='%s/list' % full_name,
+                                      version=version))
 
   def __init__(self,
                file_system,
                populate_function,
-               object_store,
-               namespace,
-               version=None):
+               file_object_store,
+               list_object_store):
     self._file_system = file_system
     self._populate_function = populate_function
-    self._object_store = object_store
-    self._namespace = 'CompiledFileSystem.' + namespace
-    if version is not None:
-      self._namespace = '%s.%s' % (self._namespace, version)
+    self._file_object_store = file_object_store
+    self._list_object_store = list_object_store
 
-  def _MakeKey(self, key):
-    return self._namespace + '.' + key
-
-  def _RecursiveList(self, files):
-    all_files = files[:]
-    dirs = {}
-    for filename in files:
+  def _RecursiveList(self, path):
+    files = []
+    for filename in self._file_system.ReadSingle(path):
       if filename.endswith('/'):
-        all_files.remove(filename)
-        dirs.update(self._file_system.Read([filename]).Get())
-    for dir_, files in dirs.iteritems():
-      all_files.extend(self._RecursiveList([dir_ + f for f in files]))
-    return all_files
+        files.extend(['%s%s' % (filename, f)
+                      for f in self._RecursiveList('%s%s' % (path, filename))])
+      else:
+        files.append(filename)
+    return files
 
   def GetFromFile(self, path, binary=False):
     """Calls |populate_function| on the contents of the file at |path|.  If
     |binary| is True then the file will be read as binary - but this will only
     apply for the first time the file is fetched; if already cached, |binary|
     will be ignored.
     """
     version = self._file_system.Stat(path).version
-    cache_entry = self._object_store.Get(self._MakeKey(path),
-                                         object_store.FILE_SYSTEM_CACHE,
-                                         time=0).Get()
+    cache_entry = self._file_object_store.Get(path, time=0).Get()
     if (cache_entry is not None) and (version == cache_entry.version):
       return cache_entry._cache_data
     cache_data = self._populate_function(
         path,
         self._file_system.ReadSingle(path, binary=binary))
-    self._object_store.Set(self._MakeKey(path),
-                           _CacheEntry(cache_data, version),
-                           object_store.FILE_SYSTEM_CACHE,
-                           time=0)
+    self._file_object_store.Set(path, _CacheEntry(cache_data, version), time=0)
     return cache_data
 
   def GetFromFileListing(self, path):
     """Calls |populate_function| on the listing of the files at |path|.
     Assumes that the path given is to a directory.
     """
     if not path.endswith('/'):
       path += '/'
     version = self._file_system.Stat(path).version
-    cache_entry = self._object_store.Get(
-        self._MakeKey(path),
-        object_store.FILE_SYSTEM_CACHE_LISTING,
-        time=0).Get()
+    cache_entry = self._list_object_store.Get(path, time=0).Get()
     if (cache_entry is not None) and (version == cache_entry.version):
         return cache_entry._cache_data
-    cache_data = self._populate_function(
-        path,
-        self._RecursiveList(
-            [path + f for f in self._file_system.ReadSingle(path)]))
-    self._object_store.Set(self._MakeKey(path),
-                           _CacheEntry(cache_data, version),
-                           object_store.FILE_SYSTEM_CACHE_LISTING,
-                           time=0)
+    cache_data = self._populate_function(path, self._RecursiveList(path))
+    self._list_object_store.Set(path, _CacheEntry(cache_data, version), time=0)
     return cache_data