Refactor APIDataSource to work with DataSourceRegistry

chrome/common/extensions/docs/server2/api_data_source.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from copy import copy
 import logging
 import os
 import posixpath
 
+from data_source import DataSource
 from environment import IsPreviewServer
 from extensions_paths import JSON_TEMPLATES, PRIVATE_TEMPLATES
+from file_system import FileNotFoundError
+from future import Future, Collect
 import third_party.json_schema_compiler.json_parse as json_parse
 import third_party.json_schema_compiler.model as model
 from environment import IsPreviewServer
 from third_party.json_schema_compiler.memoize import memoize
 
 
 def _CreateId(node, prefix):
   if node.parent is not None and not isinstance(node.parent, model.Namespace):
     return '-'.join([prefix, node.parent.simple_name, node.simple_name])
   return '-'.join([prefix, node.simple_name])
@@ skipping 39 matching lines @@
 
 
 class _JSCModel(object):
   '''Uses a Model from the JSON Schema Compiler and generates a dict that
   a Handlebar template can use for a data source.
   '''
 
   def __init__(self,
                api_name,
                api_models,
-               disable_refs,
                availability_finder,
                json_cache,
                template_cache,
                features_bundle,
                event_byname_function):
-    self._disable_refs = disable_refs
     self._availability_finder = availability_finder
     self._api_availabilities = json_cache.GetFromFile(
         posixpath.join(JSON_TEMPLATES, 'api_availabilities.json'))
     self._intro_tables = json_cache.GetFromFile(
         posixpath.join(JSON_TEMPLATES, 'intro_tables.json'))
     self._api_features = features_bundle.GetAPIFeatures()
     self._template_cache = template_cache
     self._event_byname_function = event_byname_function
     self._namespace = api_models.GetModel(api_name).Get()

[not at google - send to devlin, 2014/04/09 16:27:54]

see comment below about cron. this is one of those operations that we should be
pulling async; and adding a Cron method to APIModels would do that.

 
   def _GetLink(self, link):
     ref = link if '.' in link else (self._namespace.name + '.' + link)
     return { 'ref': ref, 'text': link, 'name': link }
 
   def ToDict(self):
     if self._namespace is None:
       return {}
     chrome_dot_name = 'chrome.%s' % self._namespace.name
     as_dict = {
       'name': self._namespace.name,
       'namespace': self._namespace.documentation_options.get('namespace',
                                                              chrome_dot_name),
       'title': self._namespace.documentation_options.get('title',
                                                          chrome_dot_name),
       'documentationOptions': self._namespace.documentation_options,
       'types': self._GenerateTypes(self._namespace.types.values()),
       'functions': self._GenerateFunctions(self._namespace.functions),
       'events': self._GenerateEvents(self._namespace.events),
       'domEvents': self._GenerateDomEvents(self._namespace.events),
       'properties': self._GenerateProperties(self._namespace.properties),
+      'introList': self._GetIntroTableList(),
+      'channelWarning': self._GetChannelWarning(),
     }
     if self._namespace.deprecated:
       as_dict['deprecated'] = self._namespace.deprecated
-    # Rendering the intro list is really expensive and there's no point doing it
-    # unless we're rending the page - and disable_refs=True implies we're not.
-    if not self._disable_refs:
-      as_dict.update({
-        'introList': self._GetIntroTableList(),
-        'channelWarning': self._GetChannelWarning(),
-      })
+
     as_dict['byName'] = _GetByNameDict(as_dict)
     return as_dict
 
   def _GetApiAvailability(self):
     return self._availability_finder.GetApiAvailability(self._namespace.name)
 
   def _GetChannelWarning(self):
     if not self._IsExperimental():
       return { self._GetApiAvailability().channel: True }
     return None
@@ skipping 58 matching lines @@
                      for condition in event.conditions],
       'actions': [self._GetLink(action) for action in event.actions],
       'supportsRules': event.supports_rules,
       'supportsListeners': event.supports_listeners,
       'properties': [],
       'id': _CreateId(event, 'event'),
       'byName': {},
     }
     self._AddCommonProperties(event_dict, event)
     # Add the Event members to each event in this object.
-    # If refs are disabled then don't worry about this, since it's only needed
-    # for rendering, and disable_refs=True implies we're not rendering.
-    if self._event_byname_function and not self._disable_refs:
+    if self._event_byname_function:
       event_dict['byName'].update(self._event_byname_function())
     # We need to create the method description for addListener based on the
     # information stored in |event|.
     if event.supports_listeners:
       callback_object = model.Function(parent=event,
                                        name='callback',
                                        json={},
                                        namespace=event.parent,
                                        origin='')
       callback_object.params = event.params
@@ skipping 252 matching lines @@
   '''
 
   def __init__(self, api_name, samples):
     self._api_name = api_name
     self._samples = samples
 
   def get(self, key):
     return self._samples.FilterSamples(key, self._api_name)
 
 
-class APIDataSource(object):
+class APIDataSource(DataSource):
   '''This class fetches and loads JSON APIs from the FileSystem passed in with
   |compiled_fs_factory|, so the APIs can be plugged into templates.
   '''
+  def __init__(self, server_instance, request):
+    self._file_system = server_instance.host_file_system_provider.GetTrunk()

[not at google - send to devlin, 2014/04/09 16:27:54]

this doesn't need to go on |self| because it's only accessed in the constructor.

+    self._json_cache = server_instance.compiled_fs_factory.ForJson(
+        self._file_system)
+    self._template_cache = server_instance.compiled_fs_factory.ForTemplates(
+        self._file_system)
+    self._availability_finder = server_instance.availability_finder
+    self._api_models = server_instance.api_models
+    self._features_bundle = server_instance.features_bundle
+    self._model_cache = server_instance.object_store_creator.Create(
+        APIDataSource)
 
-  class Factory(object):
-    def __init__(self,
-                 compiled_fs_factory,
-                 file_system,
-                 availability_finder,
-                 api_models,
-                 features_bundle,
-                 object_store_creator):
-      self._json_cache = compiled_fs_factory.ForJson(file_system)
-      self._template_cache = compiled_fs_factory.ForTemplates(file_system)
-      self._availability_finder = availability_finder
-      self._api_models = api_models
-      self._features_bundle = features_bundle
-      self._model_cache_refs = object_store_creator.Create(
-          APIDataSource, 'model-cache-refs')
-      self._model_cache_no_refs = object_store_creator.Create(
-          APIDataSource, 'model-cache-no-refs')
+    # This caches the result of _LoadEventByName.
+    self._event_byname = None
+    self._samples = server_instance.samples_data_source_factory.Create(request)
 
-      # These must be set later via the SetFooDataSourceFactory methods.
-      self._samples_data_source_factory = None
+  def _LoadEventByName(self):
+    '''All events have some members in common. We source their description
+    from Event in events.json.
+    '''
+    if self._event_byname is None:
+      self._event_byname = _GetEventByNameFromEvents(
+          self._GetSchemaModel('events'))
+    return self._event_byname
 
-      # This caches the result of _LoadEventByName.
-      self._event_byname = None
-
-    def SetSamplesDataSourceFactory(self, samples_data_source_factory):
-      self._samples_data_source_factory = samples_data_source_factory
-
-    def Create(self, request):
-      '''Creates an APIDataSource.
-      '''
-      if self._samples_data_source_factory is None:
-        # Only error if there is a request, which means this APIDataSource is
-        # actually being used to render a page.
-        if request is not None:
-          logging.error('SamplesDataSource.Factory was never set in '
-                        'APIDataSource.Factory.')
-        samples = None
-      else:
-        samples = self._samples_data_source_factory.Create(request)
-      return APIDataSource(self._GetSchemaModel, samples)
-
-    def _LoadEventByName(self):
-      '''All events have some members in common. We source their description
-      from Event in events.json.
-      '''
-      if self._event_byname is None:
-        self._event_byname = _GetEventByNameFromEvents(
-            self._GetSchemaModel('events', True))
-      return self._event_byname
-
-    def _GetModelCache(self, disable_refs):
-      if disable_refs:
-        return self._model_cache_no_refs
-      return self._model_cache_refs
-
-    def _GetSchemaModel(self, api_name, disable_refs):
-      jsc_model = self._GetModelCache(disable_refs).Get(api_name).Get()
-      if jsc_model is not None:
-        return jsc_model
-
-      jsc_model = _JSCModel(
-          api_name,
-          self._api_models,
-          disable_refs,
-          self._availability_finder,
-          self._json_cache,
-          self._template_cache,
-          self._features_bundle,
-          self._LoadEventByName).ToDict()
-
-      self._GetModelCache(disable_refs).Set(api_name, jsc_model)
+  def _GetSchemaModel(self, api_name):
+    jsc_model = self._model_cache.Get(api_name).Get()
+    if jsc_model is not None:
       return jsc_model
 
-  def __init__(self, get_schema_model, samples):
-    self._get_schema_model = get_schema_model
-    self._samples = samples
+    jsc_model = _JSCModel(
+        api_name,
+        self._api_models,
+        self._availability_finder,
+        self._json_cache,
+        self._template_cache,
+        self._features_bundle,
+        self._LoadEventByName).ToDict()
+
+    self._model_cache.Set(api_name, jsc_model)
+    return jsc_model
+
 
   def _GenerateHandlebarContext(self, handlebar_dict):
     # Parsing samples on the preview server takes seconds and doesn't add
     # anything. Don't do it.
     if not IsPreviewServer():
       handlebar_dict['samples'] = _LazySamplesGetter(
           handlebar_dict['name'],
           self._samples)
     return handlebar_dict
 
-  def get(self, api_name, disable_refs=False):

[ahernandez, 2014/04/08 17:45:06]

The only class using |disable_refs| was ReferenceResolver, but since
ReferenceResolver no longer relies on APIDataSource I decided to just remove all
the disable ref stuff.

[not at google - send to devlin, 2014/04/09 16:27:54]

Awesome.

-    return self._GenerateHandlebarContext(
-        self._get_schema_model(api_name, disable_refs))
+  def get(self, api_name):
+    return self._GenerateHandlebarContext(self._GetSchemaModel(api_name))
+
+  def Cron(self):
+    futures = []
+    try:
+      futures.extend([Future(value=self.get(api_name))

[not at google - send to devlin, 2014/04/09 16:27:54]

setting a value= for each of these defeats the purpose of making them futures.
you might as well have this as:

for api_name in self._api_models.GetNames():
  self.get(api_name)  # maybe you need the try/except around here
return Future(value=())

However, the real win will come when we can lift the asynchronous operations out
into a true Collect of async futures; typically for data sources that means:

- make an asynchronous version of .get() (say GetAsync if you want)
- the implementation of get() would be "return GetAsync().Get()"
- the implementation of the Cron method Collects those async futures.

In this particular case though, it's very complex. There is a lot of different
data that goes into here, for example,
- API models
- features
- some templates
- samples

all that should be lifted out. In the APIModels and Features cases we should
expose a Cron() method for each of those. For the "some templates" case we can
pick the templates that we actually need (intro table).

for GetAsync itself it'd be simplest to make _GetSchemaModel async on the
datastore access.

... the moral of this story, I think, is to just make the implementation of this
method "pass" for now -- since that's what it already is -- then implement the
proper efficient Cron method in a follow-up.

+                      for api_name in self._api_models.GetNames()])
+    except FileNotFoundError:
+      pass
+    return Collect(futures)