Refactor the devserver to make it easier to control caching

chrome/common/extensions/docs/server2/api_data_source.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import copy
 import logging
 import os
 
-import compiled_file_system as compiled_fs
 from file_system import FileNotFoundError
 import third_party.json_schema_compiler.json_parse as json_parse
 import third_party.json_schema_compiler.model as model
 import third_party.json_schema_compiler.idl_schema as idl_schema
 import third_party.json_schema_compiler.idl_parser as idl_parser
 
-# Increment this version when there are changes to the data stored in any of
-# the caches used by APIDataSource. This would include changes to model.py in
-# JSON schema compiler! This allows the cache to be invalidated without having
-# to flush memcache on the production server.
-_VERSION = 16
+# Increment this if the data model changes for APIDataSource.
+# This would include changes to model.py in json_schema_compiler!
+_VERSION = 17
 
 def _RemoveNoDocs(item):
   if json_parse.IsDict(item):
     if item.get('nodoc', False):
       return True
     to_remove = []
     for key, value in item.items():
       if _RemoveNoDocs(value):
         to_remove.append(key)
     for k in to_remove:
@@ skipping 230 matching lines @@
   """
   def __init__(self, api_name, samples):
     self._api_name = api_name
     self._samples = samples
 
   def get(self, key):
     return self._samples.FilterSamples(key, self._api_name)
 
 class APIDataSource(object):
   """This class fetches and loads JSON APIs from the FileSystem passed in with
-  |cache_factory|, so the APIs can be plugged into templates.
+  |compiled_fs_factory|, so the APIs can be plugged into templates.
   """
   class Factory(object):
-    def __init__(self,
-                 cache_factory,
-                 base_path):
-      self._permissions_cache = cache_factory.Create(self._LoadPermissions,
-                                                     compiled_fs.PERMS,
-                                                     version=_VERSION)
-      self._json_cache = cache_factory.Create(
+    def __init__(self, compiled_fs_factory, base_path):
+      def create_compiled_fs(fn, category):
+        return compiled_fs_factory.Create(
+            fn, APIDataSource, category=category, version=_VERSION)
+
+      self._permissions_cache = create_compiled_fs(self._LoadPermissions,
+                                                   'permissions')
+
+      self._json_cache = create_compiled_fs(
           lambda api_name, api: self._LoadJsonAPI(api, False),
-          compiled_fs.JSON,
-          version=_VERSION)
-      self._idl_cache = cache_factory.Create(
+          'json')
+      self._idl_cache = create_compiled_fs(
           lambda api_name, api: self._LoadIdlAPI(api, False),
-          compiled_fs.IDL,
-          version=_VERSION)
+          'idl')
 
       # These caches are used if an APIDataSource does not want to resolve the
       # $refs in an API. This is needed to prevent infinite recursion in
       # ReferenceResolver.
-      self._json_cache_no_refs = cache_factory.Create(
+      self._json_cache_no_refs = create_compiled_fs(
           lambda api_name, api: self._LoadJsonAPI(api, True),
-          compiled_fs.JSON_NO_REFS,
-          version=_VERSION)
-      self._idl_cache_no_refs = cache_factory.Create(
+          'json-no-refs')
+      self._idl_cache_no_refs = create_compiled_fs(
           lambda api_name, api: self._LoadIdlAPI(api, True),
-          compiled_fs.IDL_NO_REFS,
-          version=_VERSION)
-      self._idl_names_cache = cache_factory.Create(self._GetIDLNames,
-                                                   compiled_fs.IDL_NAMES,
-                                                   version=_VERSION)
-      self._names_cache = cache_factory.Create(self._GetAllNames,
-                                               compiled_fs.NAMES,
-                                               version=_VERSION)
+          'idl-no-refs')
+
+      self._idl_names_cache = create_compiled_fs(self._GetIDLNames, 'idl-names')
+      self._names_cache = create_compiled_fs(self._GetAllNames, 'names')
+
       self._base_path = base_path
 
       # These must be set later via the SetFooDataSourceFactory methods.
       self._ref_resolver_factory = None
       self._samples_data_source_factory = None
 
     def SetSamplesDataSourceFactory(self, samples_data_source_factory):
       self._samples_data_source_factory = samples_data_source_factory
 
     def SetReferenceResolverFactory(self, ref_resolver_factory):
@@ skipping 37 matching lines @@
           disable_refs).ToDict()
 
     def _LoadIdlAPI(self, api, disable_refs):
       idl = idl_parser.IDLParser().ParseData(api)
       return _JSCModel(
           idl_schema.IDLSchema(idl).process()[0],
           self._ref_resolver_factory.Create() if not disable_refs else None,
           disable_refs).ToDict()
 
     def _GetIDLNames(self, base_dir, apis):
-      return [
-        model.UnixName(os.path.splitext(api[len('%s/' % self._base_path):])[0])
-        for api in apis if api.endswith('.idl')
-      ]
+      return self._GetExtNames(apis, ['idl'])
 
     def _GetAllNames(self, base_dir, apis):
-      return [
-        model.UnixName(os.path.splitext(api[len('%s/' % self._base_path):])[0])
-        for api in apis
-      ]
+      return self._GetExtNames(apis, ['json', 'idl'])
+
+    def _GetExtNames(self, apis, exts):
+      return [model.UnixName(os.path.splitext(api)[0]) for api in apis
+              if os.path.splitext(api)[1][1:] in exts]
 
   def __init__(self,
                permissions_cache,
                json_cache,
                idl_cache,
                json_cache_no_refs,
                idl_cache_no_refs,
                names_cache,
                idl_names_cache,
                base_path,
@@ skipping 74 matching lines @@
     if self._disable_refs:
       cache, ext = (
           (self._idl_cache_no_refs, '.idl') if (unix_name in idl_names) else
           (self._json_cache_no_refs, '.json'))
     else:
       cache, ext = ((self._idl_cache, '.idl') if (unix_name in idl_names) else
                     (self._json_cache, '.json'))
     return self._GenerateHandlebarContext(
         cache.GetFromFile('%s/%s%s' % (self._base_path, unix_name, ext)),
         path)