Extensions Docs Server: Generalize $ref's to work for any schema node

chrome/common/extensions/docs/server2/api_data_source.py

Index: chrome/common/extensions/docs/server2/api_data_source.py
diff --git a/chrome/common/extensions/docs/server2/api_data_source.py b/chrome/common/extensions/docs/server2/api_data_source.py
index 49421a60f7dce381f57afb087093c8fe099cf006..f6094bc76b76225c31672732f643f80e0d0a5842 100644
--- a/chrome/common/extensions/docs/server2/api_data_source.py
+++ b/chrome/common/extensions/docs/server2/api_data_source.py
@@ -4,9 +4,9 @@
 
 import copy
 import json
+import logging
 import os
 
-from docs_server_utils import GetLinkToRefType
 import compiled_file_system as compiled_fs
 from file_system import FileNotFoundError
 import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
@@ -17,7 +17,7 @@ import third_party.json_schema_compiler.idl_parser as idl_parser
 # Increment this version when there are changes to the data stored in any of
 # the caches used by APIDataSource. This allows the cache to be invalidated
 # without having to flush memcache on the production server.
-_VERSION = 2
+_VERSION = 3
 
 def _RemoveNoDocs(item):
   if type(item) == dict:
@@ -49,17 +49,31 @@ def _FormatValue(value):
   s = str(value)
   return ','.join([s[max(0, i - 3):i] for i in range(len(s), 0, -3)][::-1])
 
-class _JscModel(object):
+class _JSCModel(object):
   """Uses a Model from the JSON Schema Compiler and generates a dict that
   a Handlebar template can use for a data source.
   """
-  def __init__(self, json):
+  def __init__(self, json, ref_resolver, disable_refs):
+    self._ref_resolver = ref_resolver
+    self._disable_refs = disable_refs
     clean_json = copy.deepcopy(json)
     if _RemoveNoDocs(clean_json):
       self._namespace = None
     else:
       self._namespace = model.Namespace(clean_json, clean_json['namespace'])
 
+  def _GetLink(self, ref):
+    if not self._disable_refs:
+      ref_data = self._ref_resolver.GetLink(self._namespace.name, ref)
+      if ref_data is not None:
+        return ref_data
+      logging.error('$ref %s could not be resolved.' % ref)
+    if '.' in ref:
+      type_name = ref.rsplit('.', 1)[-1]
+    else:
+      type_name = ref
+    return { 'href': '#type-%s' % type_name, 'text': ref }
+
   def _FormatDescription(self, description):
     if description is None or '$ref:' not in description:
       return description
@@ -76,7 +90,7 @@ class _JscModel(object):
       if not ref[-1].isalnum():
         rest = ref[-1] + rest
         ref = ref[:-1]
-      ref_dict = GetLinkToRefType(self._namespace.name, ref)
+      ref_dict = self._GetLink(ref)
       formatted_description.append('<a href="%(href)s">%(text)s</a>%(rest)s' %
           { 'href': ref_dict['href'], 'text': ref_dict['text'], 'rest': rest })
     return ''.join(formatted_description)
@@ -139,16 +153,16 @@ class _JscModel(object):
     event_dict = {
       'name': event.simple_name,
       'description': self._FormatDescription(event.description),
-      'parameters': map(self._GenerateProperty, event.params),
+      'parameters': [self._GenerateProperty(p) for p in event.params],
       'callback': self._GenerateCallback(event.callback),
-      'conditions': [GetLinkToRefType(self._namespace.name, c)
-                     for c in event.conditions],
-      'actions': [GetLinkToRefType(self._namespace.name, a)
-                     for a in event.actions],
-      'filters': map(self._GenerateProperty, event.filters),
+      'filters': [self._GenerateProperty(f) for f in event.filters],
       'supportsRules': event.supports_rules,
       'id': _CreateId(event, 'event')
     }
+    event_dict['conditions'] = [self._GetLink(condition)
+                                for condition in event.conditions]
+    event_dict['actions'] = [self._GetLink(action)
+                             for action in event.actions]

[not at google - send to devlin, 2012/11/05 19:47:50]

these can be inlined again?

[cduvall, 2012/11/06 00:58:54]

Done.

     if (event.parent is not None and
         not isinstance(event.parent, model.Namespace)):
       event_dict['parent_name'] = event.parent.simple_name
@@ -186,8 +200,8 @@ class _JscModel(object):
       'optional': property_.optional,
       'description': self._FormatDescription(property_.description),
       'properties': self._GenerateProperties(property_.properties),
-      'parameters': [],
       'functions': self._GenerateFunctions(property_.functions),
+      'parameters': [],
       'returns': None,
       'id': _CreateId(property_, 'property')
     }
@@ -211,15 +225,14 @@ class _JscModel(object):
 
   def _RenderTypeInformation(self, property_, dst_dict):
     if property_.type_ == model.PropertyType.CHOICES:
-      dst_dict['choices'] = map(self._GenerateProperty,
-                                property_.choices.values())
+      dst_dict['choices'] = [self._GenerateProperty(c)
+                             for c in property_.choices.values()]
       # We keep track of which is last for knowing when to add "or" between
       # choices in templates.
       if len(dst_dict['choices']) > 0:
         dst_dict['choices'][-1]['last'] = True
     elif property_.type_ == model.PropertyType.REF:
-      dst_dict['link'] = GetLinkToRefType(self._namespace.name,
-                                          property_.ref_type)
+      dst_dict['link'] = self._GetLink(property_.ref_type)
     elif property_.type_ == model.PropertyType.ARRAY:
       dst_dict['array'] = self._GenerateProperty(property_.item_type)
     elif property_.type_ == model.PropertyType.ENUM:
@@ -249,57 +262,121 @@ class APIDataSource(object):
   |cache_factory|, so the APIs can be plugged into templates.
   """
   class Factory(object):
-    def __init__(self, cache_factory, base_path, samples_factory):
+    def __init__(self,
+                 cache_factory,
+                 base_path):
       self._permissions_cache = cache_factory.Create(self._LoadPermissions,
                                                      compiled_fs.PERMS,
                                                      version=_VERSION)
-      self._json_cache = cache_factory.Create(self._LoadJsonAPI,
-                                              compiled_fs.JSON,
-                                              version=_VERSION)
-      self._idl_cache = cache_factory.Create(self._LoadIdlAPI,
-                                             compiled_fs.IDL,
-                                             version=_VERSION)
+      self._json_cache = cache_factory.Create(
+          lambda api: self._LoadJsonAPI(api, False),
+          compiled_fs.JSON,
+          version=_VERSION)
+      self._idl_cache = cache_factory.Create(
+          lambda api: self._LoadIdlAPI(api, False),
+          compiled_fs.IDL,
+          version=_VERSION)
+      self._json_cache_no_refs = cache_factory.Create(

[not at google - send to devlin, 2012/11/05 19:47:50]

Nice. Quick comment on why having separate caches with refs vs no refs is
necessary?

[cduvall, 2012/11/06 00:58:54]

Done.

+          lambda api: self._LoadJsonAPI(api, True),
+          compiled_fs.JSON_NO_REFS,
+          version=_VERSION)
+      self._idl_cache_no_refs = cache_factory.Create(
+          lambda api: self._LoadIdlAPI(api, True),
+          compiled_fs.IDL_NO_REFS,
+          version=_VERSION)
       self._idl_names_cache = cache_factory.Create(self._GetIDLNames,
                                                    compiled_fs.IDL_NAMES,
                                                    version=_VERSION)
-      self._samples_factory = samples_factory
+      self._names_cache = cache_factory.Create(self._GetAllNames,
+                                               compiled_fs.NAMES,
+                                               version=_VERSION)
       self._base_path = base_path
-
-    def Create(self, request):
+      self._ref_resolver_factory = None

[not at google - send to devlin, 2012/11/05 19:47:50]

# These must be set later via the SetFooDataSourceFactory methods.

[cduvall, 2012/11/06 00:58:54]

Done.

+      self._samples_data_source_factory = None
+
+    def SetSamplesDataSourceFactory(self, samples_data_source_factory):
+      self._samples_data_source_factory = samples_data_source_factory
+
+    def SetReferenceResolverFactory(self, ref_resolver_factory):
+      self._ref_resolver_factory = ref_resolver_factory
+
+    def Create(self, request, disable_refs=False):
+      """Create an APIDataSource. |disable_refs| specifies whether $ref's in
+      APIs being processed by the |ToDict| method of _JSCModel follows $ref's
+      in the API. This prevents endless recursion in ReferenceResolver.
+      """
+      if self._samples_data_source_factory is None:

[cduvall, 2012/11/03 01:30:05]

I have this chunk here because SamplesDataSource needs a ReferenceResolver, and
ReferenceResolver needs an APIDataSource. So we need to be able to make an
APIDataSource without the factory having a SampelsDataSource.Factory.

The same applies below, with ReferenceResolver and APIDataSource being
interdependent.

+        # Only error if there is a request, which means this APIDataSource is
+        # actually being used to render a page.
+        if request is not None:
+          logging.error('SamplesDataSource.Factory was never set in '
+                        'APIDataSource.Factory.')
+        samples = None
+      else:
+        samples = self._samples_data_source_factory.Create(request)
+      if not disable_refs and self._ref_resolver_factory is None:
+        logging.error('ReferenceResolver.Factory was never set in '
+                      'APIDataSource.Factory.')
       return APIDataSource(self._permissions_cache,
                            self._json_cache,
                            self._idl_cache,
+                           self._json_cache_no_refs,
+                           self._idl_cache_no_refs,
+                           self._names_cache,
                            self._idl_names_cache,
                            self._base_path,
-                           self._samples_factory.Create(request))
+                           samples,
+                           disable_refs)
 
     def _LoadPermissions(self, json_str):
       return json.loads(json_comment_eater.Nom(json_str))
 
-    def _LoadJsonAPI(self, api):
-      return _JscModel(json.loads(json_comment_eater.Nom(api))[0])
+    def _LoadJsonAPI(self, api, disable_refs):
+      return _JSCModel(
+          json.loads(json_comment_eater.Nom(api))[0],
+          self._ref_resolver_factory.Create() if not disable_refs else None,
+          disable_refs).ToDict()
 
-    def _LoadIdlAPI(self, api):
+    def _LoadIdlAPI(self, api, disable_refs):
       idl = idl_parser.IDLParser().ParseData(api)
-      return _JscModel(idl_schema.IDLSchema(idl).process()[0])
+      return _JSCModel(
+          idl_schema.IDLSchema(idl).process()[0],
+          self._ref_resolver_factory.Create() if not disable_refs else None,
+          disable_refs).ToDict()
 
     def _GetIDLNames(self, apis):
-      return [model.UnixName(os.path.splitext(api.split('/')[-1])[0])
-              for api in apis if api.endswith('.idl')]
+      return [
+        model.UnixName(os.path.splitext(api[len('%s/' % self._base_path):])[0])
+        for api in apis if api.endswith('.idl')
+      ]
+
+    def _GetAllNames(self, apis):
+      return [
+        model.UnixName(os.path.splitext(api[len('%s/' % self._base_path):])[0])
+        for api in apis
+      ]
 
   def __init__(self,
                permissions_cache,
                json_cache,
                idl_cache,
+               json_cache_no_refs,
+               idl_cache_no_refs,
+               names_cache,
                idl_names_cache,
                base_path,
-               samples):
+               samples,
+               disable_refs):
     self._base_path = base_path
     self._permissions_cache = permissions_cache
     self._json_cache = json_cache
     self._idl_cache = idl_cache
+    self._json_cache_no_refs = json_cache_no_refs
+    self._idl_cache_no_refs = idl_cache_no_refs
+    self._names_cache = names_cache
     self._idl_names_cache = idl_names_cache
     self._samples = samples
+    self._disable_refs = disable_refs
 
   def _GetPermsFromFile(self, filename):
     try:
@@ -321,23 +398,39 @@ class APIDataSource(object):
       api_perms[api_perms['channel']] = True
     return api_perms
 
-  def _GenerateHandlebarContext(self, handlebar, path):
+  def _GenerateHandlebarContext(self, handlebar_dict, path):
     return_dict = {
       'permissions': self._GetFeature(path),
       'samples': _LazySamplesGetter(path, self._samples)
     }
-    return_dict.update(handlebar.ToDict())
+    return_dict.update(handlebar_dict)
     return return_dict
 
-  def __getitem__(self, key):
-    return self.get(key)
+  def _GetAsSubdirectory(self, name):
+    if name.startswith('experimental_'):
+      parts = name[len('experimental_'):].split('_', 1)
+      parts[1] = 'experimental_%s' % parts[1]
+      return '/'.join(parts)
+    return name.replace('_', '/', 1)
 
   def get(self, key):
-    path, ext = os.path.splitext(key)
+    if key.endswith('.html') or key.endswith('.json') or key.endswith('.idl'):
+      path, ext = os.path.splitext(key)
+    else:
+      path = key
     unix_name = model.UnixName(path)
     idl_names = self._idl_names_cache.GetFromFileListing(self._base_path)
-    cache, ext = ((self._idl_cache, '.idl') if (unix_name in idl_names) else
-                  (self._json_cache, '.json'))
+    names = self._names_cache.GetFromFileListing(self._base_path)
+    if unix_name not in names and self._GetAsSubdirectory(unix_name) in names:
+      unix_name = self._GetAsSubdirectory(unix_name)
+
+    if self._disable_refs:
+      cache, ext = (
+          (self._idl_cache_no_refs, '.idl') if (unix_name in idl_names) else
+          (self._json_cache_no_refs, '.json'))
+    else:
+      cache, ext = ((self._idl_cache, '.idl') if (unix_name in idl_names) else
+                    (self._json_cache, '.json'))
     return self._GenerateHandlebarContext(
         cache.GetFromFile('%s/%s%s' % (self._base_path, unix_name, ext)),
         path)