Extensions Docs Server: Generalize $ref's to work for any schema node

chrome/common/extensions/docs/server2/api_data_source.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import copy
 import json
+import logging
 import os
 
-from docs_server_utils import GetLinkToRefType
 import compiled_file_system as compiled_fs
 from file_system import FileNotFoundError
+from reference_resolver import ReferenceResolver
 import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
 import third_party.json_schema_compiler.model as model
 import third_party.json_schema_compiler.idl_schema as idl_schema
 import third_party.json_schema_compiler.idl_parser as idl_parser
 
 # Increment this version when there are changes to the data stored in any of
 # the caches used by APIDataSource. This allows the cache to be invalidated
 # without having to flush memcache on the production server.
-_VERSION = 2
+_VERSION = 3
 
 def _RemoveNoDocs(item):
   if type(item) == dict:
     if item.get('nodoc', False):
       return True
     to_remove = []
     for key, value in item.items():
       if _RemoveNoDocs(value):
         to_remove.append(key)
     for k in to_remove:
@@ skipping 11 matching lines @@
   if node.parent is not None and not isinstance(node.parent, model.Namespace):
     return '-'.join([prefix, node.parent.simple_name, node.simple_name])
   return '-'.join([prefix, node.simple_name])
 
 def _FormatValue(value):
   """Inserts commas every three digits for integer values. It is magic.
   """
   s = str(value)
   return ','.join([s[max(0, i - 3):i] for i in range(len(s), 0, -3)][::-1])
 
-class _JscModel(object):
+class _JSCModel(object):
   """Uses a Model from the JSON Schema Compiler and generates a dict that
   a Handlebar template can use for a data source.
   """
   def __init__(self, json):
     clean_json = copy.deepcopy(json)
     if _RemoveNoDocs(clean_json):
       self._namespace = None
     else:
       self._namespace = model.Namespace(clean_json, clean_json['namespace'])
 
-  def _FormatDescription(self, description):
-    if description is None or '$ref:' not in description:
+  def _GetLinkToRefType(self, ref, ref_resolver):

[not at google - send to devlin, 2012/11/02 17:26:48]

Rename method similar to rename suggestion in ReferenceResolver.

[cduvall, 2012/11/03 01:30:05]

Done.

+    ref_data = ref_resolver.GetLinkToRefType(self._namespace.name, ref)
+    if ref_data is None:

[not at google - send to devlin, 2012/11/02 17:26:48]

nit: invert condition.

[cduvall, 2012/11/03 01:30:05]

Done.

+      logging.error('$ref %s could not be resolved.' % ref)
+      if '.' in ref:
+        type_name = ref.rsplit('.', 1)[-1]
+      else:
+        type_name = ref
+      return { 'href': '#type-%s' % type_name, 'text': ref }
+    return ref_data
+
+  def _FormatDescription(self, description, ref_resolver):
+    if (description is None or
+        '$ref:' not in description or
+        ref_resolver is None):
       return description
     refs = description.split('$ref:')
     formatted_description = [refs[0]]
     for ref in refs[1:]:
       parts = ref.split(' ', 1)
       if len(parts) == 1:
         ref = parts[0]
         rest = ''
       else:
         ref, rest = parts
         rest = ' ' + rest
       if not ref[-1].isalnum():
         rest = ref[-1] + rest
         ref = ref[:-1]
-      ref_dict = GetLinkToRefType(self._namespace.name, ref)
+      ref_dict = self._GetLinkToRefType(ref, ref_resolver)
       formatted_description.append('<a href="%(href)s">%(text)s</a>%(rest)s' %
           { 'href': ref_dict['href'], 'text': ref_dict['text'], 'rest': rest })
     return ''.join(formatted_description)
 
-  def ToDict(self):
+  def ToDict(self, ref_resolver):

[not at google - send to devlin, 2012/11/02 17:26:48]

Consider introducing a DictBuilder class that is constructed with the JSC model
and the ref resolver as its dependencies, rather than needing to thread
ref_resolver everywhere.

[cduvall, 2012/11/03 01:30:05]

I redid JSCModel so that it acts as a DictBuilder. Now the whole output of
ToDict is cached, so this should speed things up a little.

     if self._namespace is None:
       return {}
     return {
       'name': self._namespace.name,
-      'types':  [self._GenerateType(t) for t in self._namespace.types.values()
+      'types':  [self._GenerateType(t, ref_resolver)
+                 for t in self._namespace.types.values()
                  if t.type_ != model.PropertyType.ADDITIONAL_PROPERTIES],
-      'functions': self._GenerateFunctions(self._namespace.functions),
-      'events': self._GenerateEvents(self._namespace.events),
-      'properties': self._GenerateProperties(self._namespace.properties)
+      'functions': self._GenerateFunctions(self._namespace.functions,
+                                           ref_resolver),
+      'events': self._GenerateEvents(self._namespace.events,
+                                     ref_resolver),
+      'properties': self._GenerateProperties(self._namespace.properties,
+                                             ref_resolver)
     }
 
-  def _GenerateType(self, type_):
+  def _GenerateType(self, type_, ref_resolver):
     type_dict = {
       'name': type_.simple_name,
-      'description': self._FormatDescription(type_.description),
-      'properties': self._GenerateProperties(type_.properties),
-      'functions': self._GenerateFunctions(type_.functions),
-      'events': self._GenerateEvents(type_.events),
+      'description': self._FormatDescription(type_.description, ref_resolver),
+      'properties': self._GenerateProperties(type_.properties, ref_resolver),
+      'functions': self._GenerateFunctions(type_.functions, ref_resolver),
+      'events': self._GenerateEvents(type_.events, ref_resolver),
       'id': _CreateId(type_, 'type')
     }
-    self._RenderTypeInformation(type_, type_dict)
+    self._RenderTypeInformation(type_, type_dict, ref_resolver)
     return type_dict
 
-  def _GenerateFunctions(self, functions):
-    return [self._GenerateFunction(f) for f in functions.values()]
+  def _GenerateFunctions(self, functions, ref_resolver):
+    return [self._GenerateFunction(f, ref_resolver) for f in functions.values()]
 
-  def _GenerateFunction(self, function):
+  def _GenerateFunction(self, function, ref_resolver):
     function_dict = {
       'name': function.simple_name,
-      'description': self._FormatDescription(function.description),
-      'callback': self._GenerateCallback(function.callback),
+      'description': self._FormatDescription(function.description,
+                                             ref_resolver),
+      'callback': self._GenerateCallback(function.callback, ref_resolver),
       'parameters': [],
       'returns': None,
       'id': _CreateId(function, 'method')
     }
     if (function.parent is not None and
         not isinstance(function.parent, model.Namespace)):
       function_dict['parent_name'] = function.parent.simple_name
     else:
       function_dict['parent_name'] = None
     if function.returns:
-      function_dict['returns'] = self._GenerateProperty(function.returns)
+      function_dict['returns'] = self._GenerateProperty(function.returns,
+                                                        ref_resolver)
     for param in function.params:
-      function_dict['parameters'].append(self._GenerateProperty(param))
+      function_dict['parameters'].append(self._GenerateProperty(param,
+                                                                ref_resolver))
     if function_dict['callback']:
       function_dict['parameters'].append(function_dict['callback'])
     if len(function_dict['parameters']) > 0:
       function_dict['parameters'][-1]['last'] = True
     return function_dict
 
-  def _GenerateEvents(self, events):
-    return [self._GenerateEvent(e) for e in events.values()]
+  def _GenerateEvents(self, events, ref_resolver):
+    return [self._GenerateEvent(e, ref_resolver) for e in events.values()]
 
-  def _GenerateEvent(self, event):
+  def _GenerateEvent(self, event, ref_resolver):
     event_dict = {
       'name': event.simple_name,
-      'description': self._FormatDescription(event.description),
-      'parameters': map(self._GenerateProperty, event.params),
-      'callback': self._GenerateCallback(event.callback),
-      'conditions': [GetLinkToRefType(self._namespace.name, c)
-                     for c in event.conditions],
-      'actions': [GetLinkToRefType(self._namespace.name, a)
-                     for a in event.actions],
-      'filters': map(self._GenerateProperty, event.filters),
+      'description': self._FormatDescription(event.description,
+                                             ref_resolver),
+      'parameters': [self._GenerateProperty(p, ref_resolver)
+                     for p in event.params],
+      'callback': self._GenerateCallback(event.callback, ref_resolver),
+      'filters': [self._GenerateProperty(f, ref_resolver)
+                  for f in event.filters],
       'supportsRules': event.supports_rules,
       'id': _CreateId(event, 'event')
     }
+    if ref_resolver is not None:
+      event_dict['conditions'] = [self._GetLinkToRefType(condition,
+                                                         ref_resolver)
+                                  for condition in event.conditions]
+      event_dict['actions'] = [self._GetLinkToRefType(action, ref_resolver)
+                               for action in event.actions]
     if (event.parent is not None and
         not isinstance(event.parent, model.Namespace)):
       event_dict['parent_name'] = event.parent.simple_name
     else:
       event_dict['parent_name'] = None
     if event_dict['callback']:
       event_dict['parameters'].append(event_dict['callback'])
     if len(event_dict['parameters']) > 0:
       event_dict['parameters'][-1]['last'] = True
     return event_dict
 
-  def _GenerateCallback(self, callback):
+  def _GenerateCallback(self, callback, ref_resolver):
     if not callback:
       return None
     callback_dict = {
       'name': callback.simple_name,
-      'description': self._FormatDescription(callback.description),
+      'description': self._FormatDescription(callback.description,
+                                             ref_resolver),
       'simple_type': {'simple_type': 'function'},
       'optional': callback.optional,
       'parameters': []
     }
     for param in callback.params:
-      callback_dict['parameters'].append(self._GenerateProperty(param))
+      callback_dict['parameters'].append(
+          self._GenerateProperty(param, ref_resolver))
     if (len(callback_dict['parameters']) > 0):
       callback_dict['parameters'][-1]['last'] = True
     return callback_dict
 
-  def _GenerateProperties(self, properties):
-    return [self._GenerateProperty(v) for v in properties.values()
+  def _GenerateProperties(self, properties, ref_resolver):
+    return [self._GenerateProperty(v, ref_resolver) for v in properties.values()
             if v.type_ != model.PropertyType.ADDITIONAL_PROPERTIES]
 
-  def _GenerateProperty(self, property_):
+  def _GenerateProperty(self, property_, ref_resolver):
     property_dict = {
       'name': property_.simple_name,
       'optional': property_.optional,
-      'description': self._FormatDescription(property_.description),
-      'properties': self._GenerateProperties(property_.properties),
+      'description': self._FormatDescription(property_.description,
+                                             ref_resolver),
+      'properties': self._GenerateProperties(property_.properties,
+                                             ref_resolver),
+      'functions': self._GenerateFunctions(property_.functions, ref_resolver),
       'parameters': [],
-      'functions': self._GenerateFunctions(property_.functions),
       'returns': None,
       'id': _CreateId(property_, 'property')
     }
     for param in property_.params:
-      property_dict['parameters'].append(self._GenerateProperty(param))
+      property_dict['parameters'].append(self._GenerateProperty(param,
+                                                                ref_resolver))
     if property_.returns:
       property_dict['returns'] = self._GenerateProperty(property_.returns)
     if (property_.parent is not None and
         not isinstance(property_.parent, model.Namespace)):
       property_dict['parent_name'] = property_.parent.simple_name
     else:
       property_dict['parent_name'] = None
     if property_.has_value:
       if isinstance(property_.value, int):
         property_dict['value'] = _FormatValue(property_.value)
       else:
         property_dict['value'] = property_.value
     else:
-      self._RenderTypeInformation(property_, property_dict)
+      self._RenderTypeInformation(property_, property_dict, ref_resolver)
     return property_dict
 
-  def _RenderTypeInformation(self, property_, dst_dict):
+  def _RenderTypeInformation(self, property_, dst_dict, ref_resolver):
     if property_.type_ == model.PropertyType.CHOICES:
-      dst_dict['choices'] = map(self._GenerateProperty,
-                                property_.choices.values())
+      dst_dict['choices'] = [self._GenerateProperty(c, ref_resolver)
+                             for c in property_.choices.values()]
       # We keep track of which is last for knowing when to add "or" between
       # choices in templates.
       if len(dst_dict['choices']) > 0:
         dst_dict['choices'][-1]['last'] = True
     elif property_.type_ == model.PropertyType.REF:
-      dst_dict['link'] = GetLinkToRefType(self._namespace.name,
-                                          property_.ref_type)
+      if ref_resolver is not None:
+        dst_dict['link'] = self._GetLinkToRefType(property_.ref_type,
+                                                  ref_resolver)
     elif property_.type_ == model.PropertyType.ARRAY:
-      dst_dict['array'] = self._GenerateProperty(property_.item_type)
+      dst_dict['array'] = self._GenerateProperty(property_.item_type,
+                                                 ref_resolver)
     elif property_.type_ == model.PropertyType.ENUM:
       dst_dict['enum_values'] = []
       for enum_value in property_.enum_values:
         dst_dict['enum_values'].append({'name': enum_value})
       if len(dst_dict['enum_values']) > 0:
         dst_dict['enum_values'][-1]['last'] = True
     elif property_.instance_of is not None:
       dst_dict['simple_type'] = property_.instance_of.lower()
     else:
       dst_dict['simple_type'] = property_.type_.name.lower()
 
 class _LazySamplesGetter(object):
   """This class is needed so that an extensions API page does not have to fetch
   the apps samples page and vice versa.
   """
   def __init__(self, api_name, samples):
     self._api_name = api_name
     self._samples = samples
 
   def get(self, key):
     return self._samples.FilterSamples(key, self._api_name)
 
 class APIDataSource(object):
   """This class fetches and loads JSON APIs from the FileSystem passed in with
   |cache_factory|, so the APIs can be plugged into templates.
   """
   class Factory(object):
-    def __init__(self, cache_factory, base_path, samples_factory):
+    def __init__(self,
+                 cache_factory,
+                 base_path,
+                 api_list_data_source):

[not at google - send to devlin, 2012/11/02 17:26:48]

As I mentioned in handler, should be passing the factory here rather than a
concrete class.

If you make a ReferenceResolver.Factory class like I suggest in
SamplesDataSource I believe this problem will go away.

[cduvall, 2012/11/03 01:30:05]

Done.

       self._permissions_cache = cache_factory.Create(self._LoadPermissions,
                                                      compiled_fs.PERMS,
                                                      version=_VERSION)
       self._json_cache = cache_factory.Create(self._LoadJsonAPI,
                                               compiled_fs.JSON,
                                               version=_VERSION)
       self._idl_cache = cache_factory.Create(self._LoadIdlAPI,
                                              compiled_fs.IDL,
                                              version=_VERSION)
       self._idl_names_cache = cache_factory.Create(self._GetIDLNames,
                                                    compiled_fs.IDL_NAMES,
                                                    version=_VERSION)
-      self._samples_factory = samples_factory
+      self._names_cache = cache_factory.Create(self._GetAllNames,
+                                               compiled_fs.NAMES,
+                                               version=_VERSION)
       self._base_path = base_path
+      self._ref_resolver = ReferenceResolver(
+          self.Create(None, None, resolve_refs=False),
+          api_list_data_source)
 
-    def Create(self, request):
+    def Create(self, request, samples_data_source, resolve_refs=True):

[not at google - send to devlin, 2012/11/02 17:26:48]

It's nice for optional arguments to be False by default. Perhaps disable_refs or
something?

[not at google - send to devlin, 2012/11/02 17:26:48]

Having a separate SetSamplesDataSourceFactory method on APIDataSource.Factory
(called after its construction), rather than injecting this at creation time, is
a better way to avoid circular dependencies.

Don't forget to assert that it's been set before Create is called, though.

[cduvall, 2012/11/03 01:30:05]

Done.

[cduvall, 2012/11/03 01:30:05]

Done.

+      """Create an APIDataSource. |resolve_refs| specifies whether $ref's in
+      APIs being processed by the |ToDict| method of _JSCModel follows $ref's
+      in the API. This prevents endless recursion in ReferenceResolver.
+      """
       return APIDataSource(self._permissions_cache,
                            self._json_cache,
                            self._idl_cache,
+                           self._names_cache,
                            self._idl_names_cache,
                            self._base_path,
-                           self._samples_factory.Create(request))
+                           samples_data_source,
+                           self._ref_resolver if resolve_refs else None)

[not at google - send to devlin, 2012/11/02 17:26:48]

nit: push this logic into where ToDict is called (store both ref_resolver and
resolve_refs). 

[cduvall, 2012/11/03 01:30:05]

Done.

 
     def _LoadPermissions(self, json_str):
       return json.loads(json_comment_eater.Nom(json_str))
 
     def _LoadJsonAPI(self, api):
-      return _JscModel(json.loads(json_comment_eater.Nom(api))[0])
+      return _JSCModel(json.loads(json_comment_eater.Nom(api))[0])
 
     def _LoadIdlAPI(self, api):
       idl = idl_parser.IDLParser().ParseData(api)
-      return _JscModel(idl_schema.IDLSchema(idl).process()[0])
+      return _JSCModel(idl_schema.IDLSchema(idl).process()[0])
 
     def _GetIDLNames(self, apis):
-      return [model.UnixName(os.path.splitext(api.split('/')[-1])[0])
-              for api in apis if api.endswith('.idl')]
+      return [
+        model.UnixName(os.path.splitext(api[len('%s/' % self._base_path):])[0])
+        for api in apis if api.endswith('.idl')
+      ]
+
+    def _GetAllNames(self, apis):
+      return [
+        model.UnixName(os.path.splitext(api[len('%s/' % self._base_path):])[0])
+        for api in apis
+      ]
 
   def __init__(self,
                permissions_cache,
                json_cache,
                idl_cache,
+               names_cache,
                idl_names_cache,
                base_path,
-               samples):
+               samples,
+               ref_resolver):
     self._base_path = base_path
     self._permissions_cache = permissions_cache
     self._json_cache = json_cache
     self._idl_cache = idl_cache
+    self._names_cache = names_cache
     self._idl_names_cache = idl_names_cache
     self._samples = samples
+    self._ref_resolver = ref_resolver
 
   def _GetPermsFromFile(self, filename):
     try:
       perms = self._permissions_cache.GetFromFile('%s/%s' %
           (self._base_path, filename))
       return dict((model.UnixName(k), v) for k, v in perms.iteritems())
     except FileNotFoundError:
       return {}
 
   def _GetFeature(self, path):
     # Remove 'experimental_' from path name to match the keys in
     # _permissions_features.json.
     path = model.UnixName(path.replace('experimental_', ''))
     for filename in ['_permission_features.json', '_manifest_features.json']:
       api_perms = self._GetPermsFromFile(filename).get(path, None)
       if api_perms is not None:
         break
     if api_perms and api_perms['channel'] in ('trunk', 'dev', 'beta'):
       api_perms[api_perms['channel']] = True
     return api_perms
 
   def _GenerateHandlebarContext(self, handlebar, path):
     return_dict = {
       'permissions': self._GetFeature(path),
       'samples': _LazySamplesGetter(path, self._samples)
     }
-    return_dict.update(handlebar.ToDict())
+    return_dict.update(handlebar.ToDict(self._ref_resolver))
     return return_dict
 
+  def _GetAsSubdirectory(self, name):
+    if name.startswith('experimental_'):
+      parts = name[len('experimental_'):].split('_', 1)
+      parts[1] = 'experimental_%s' % parts[1]
+      return '/'.join(parts)
+    return name.replace('_', '/', 1)
+
   def __getitem__(self, key):
     return self.get(key)
 
   def get(self, key):
-    path, ext = os.path.splitext(key)
+    if key.endswith('.html') or key.endswith('.json') or key.endswith('.idl'):
+      path, ext = os.path.splitext(key)
+    else:
+      path = key
     unix_name = model.UnixName(path)
     idl_names = self._idl_names_cache.GetFromFileListing(self._base_path)
+    names = self._names_cache.GetFromFileListing(self._base_path)
+    if unix_name not in names and self._GetAsSubdirectory(unix_name) in names:
+      unix_name = self._GetAsSubdirectory(unix_name)
+
     cache, ext = ((self._idl_cache, '.idl') if (unix_name in idl_names) else
                   (self._json_cache, '.json'))
     return self._GenerateHandlebarContext(
         cache.GetFromFile('%s/%s%s' % (self._base_path, unix_name, ext)),
         path)