Extensions Docs Server: Generalize $ref's to work for any schema node

chrome/common/extensions/docs/server2/api_data_source.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import copy
 import json
+import logging
 import os
 
-from docs_server_utils import GetLinkToRefType
 import compiled_file_system as compiled_fs
 from file_system import FileNotFoundError
 import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
 import third_party.json_schema_compiler.model as model
 import third_party.json_schema_compiler.idl_schema as idl_schema
 import third_party.json_schema_compiler.idl_parser as idl_parser
 
 # Increment this version when there are changes to the data stored in any of
 # the caches used by APIDataSource. This allows the cache to be invalidated
 # without having to flush memcache on the production server.
-_VERSION = 2
+_VERSION = 3
 
 def _RemoveNoDocs(item):
   if type(item) == dict:
     if item.get('nodoc', False):
       return True
     to_remove = []
     for key, value in item.items():
       if _RemoveNoDocs(value):
         to_remove.append(key)
     for k in to_remove:
@@ skipping 11 matching lines @@
   if node.parent is not None and not isinstance(node.parent, model.Namespace):
     return '-'.join([prefix, node.parent.simple_name, node.simple_name])
   return '-'.join([prefix, node.simple_name])
 
 def _FormatValue(value):
   """Inserts commas every three digits for integer values. It is magic.
   """
   s = str(value)
   return ','.join([s[max(0, i - 3):i] for i in range(len(s), 0, -3)][::-1])
 
-class _JscModel(object):
+class _JSCModel(object):
   """Uses a Model from the JSON Schema Compiler and generates a dict that
   a Handlebar template can use for a data source.
   """
-  def __init__(self, json):
+  def __init__(self, json, ref_resolver, disable_refs):
+    self._ref_resolver = ref_resolver
+    self._disable_refs = disable_refs
     clean_json = copy.deepcopy(json)
     if _RemoveNoDocs(clean_json):
       self._namespace = None
     else:
       self._namespace = model.Namespace(clean_json, clean_json['namespace'])
 
+  def _GetLink(self, ref):
+    if not self._disable_refs:
+      ref_data = self._ref_resolver.GetLink(self._namespace.name, ref)
+      if ref_data is not None:
+        return ref_data
+      logging.error('$ref %s could not be resolved.' % ref)
+    if '.' in ref:
+      type_name = ref.rsplit('.', 1)[-1]
+    else:
+      type_name = ref
+    return { 'href': '#type-%s' % type_name, 'text': ref }
+
   def _FormatDescription(self, description):
     if description is None or '$ref:' not in description:
       return description
     refs = description.split('$ref:')
     formatted_description = [refs[0]]
     for ref in refs[1:]:
       parts = ref.split(' ', 1)
       if len(parts) == 1:
         ref = parts[0]
         rest = ''
       else:
         ref, rest = parts
         rest = ' ' + rest
       if not ref[-1].isalnum():
         rest = ref[-1] + rest
         ref = ref[:-1]
-      ref_dict = GetLinkToRefType(self._namespace.name, ref)
+      ref_dict = self._GetLink(ref)
       formatted_description.append('<a href="%(href)s">%(text)s</a>%(rest)s' %
           { 'href': ref_dict['href'], 'text': ref_dict['text'], 'rest': rest })
     return ''.join(formatted_description)
 
   def ToDict(self):
     if self._namespace is None:
       return {}
     return {
       'name': self._namespace.name,
       'types':  [self._GenerateType(t) for t in self._namespace.types.values()
@@ skipping 42 matching lines @@
       function_dict['parameters'][-1]['last'] = True
     return function_dict
 
   def _GenerateEvents(self, events):
     return [self._GenerateEvent(e) for e in events.values()]
 
   def _GenerateEvent(self, event):
     event_dict = {
       'name': event.simple_name,
       'description': self._FormatDescription(event.description),
-      'parameters': map(self._GenerateProperty, event.params),
+      'parameters': [self._GenerateProperty(p) for p in event.params],
       'callback': self._GenerateCallback(event.callback),
-      'conditions': [GetLinkToRefType(self._namespace.name, c)
-                     for c in event.conditions],
-      'actions': [GetLinkToRefType(self._namespace.name, a)
-                     for a in event.actions],
-      'filters': map(self._GenerateProperty, event.filters),
+      'filters': [self._GenerateProperty(f) for f in event.filters],
       'supportsRules': event.supports_rules,
       'id': _CreateId(event, 'event')
     }
+    event_dict['conditions'] = [self._GetLink(condition)
+                                for condition in event.conditions]
+    event_dict['actions'] = [self._GetLink(action)
+                             for action in event.actions]

[not at google - send to devlin, 2012/11/05 19:47:50]

these can be inlined again?

[cduvall, 2012/11/06 00:58:54]

Done.

     if (event.parent is not None and
         not isinstance(event.parent, model.Namespace)):
       event_dict['parent_name'] = event.parent.simple_name
     else:
       event_dict['parent_name'] = None
     if event_dict['callback']:
       event_dict['parameters'].append(event_dict['callback'])
     if len(event_dict['parameters']) > 0:
       event_dict['parameters'][-1]['last'] = True
     return event_dict
@@ skipping 17 matching lines @@
   def _GenerateProperties(self, properties):
     return [self._GenerateProperty(v) for v in properties.values()
             if v.type_ != model.PropertyType.ADDITIONAL_PROPERTIES]
 
   def _GenerateProperty(self, property_):
     property_dict = {
       'name': property_.simple_name,
       'optional': property_.optional,
       'description': self._FormatDescription(property_.description),
       'properties': self._GenerateProperties(property_.properties),
+      'functions': self._GenerateFunctions(property_.functions),
       'parameters': [],
-      'functions': self._GenerateFunctions(property_.functions),
       'returns': None,
       'id': _CreateId(property_, 'property')
     }
     for param in property_.params:
       property_dict['parameters'].append(self._GenerateProperty(param))
     if property_.returns:
       property_dict['returns'] = self._GenerateProperty(property_.returns)
     if (property_.parent is not None and
         not isinstance(property_.parent, model.Namespace)):
       property_dict['parent_name'] = property_.parent.simple_name
     else:
       property_dict['parent_name'] = None
     if property_.has_value:
       if isinstance(property_.value, int):
         property_dict['value'] = _FormatValue(property_.value)
       else:
         property_dict['value'] = property_.value
     else:
       self._RenderTypeInformation(property_, property_dict)
     return property_dict
 
   def _RenderTypeInformation(self, property_, dst_dict):
     if property_.type_ == model.PropertyType.CHOICES:
-      dst_dict['choices'] = map(self._GenerateProperty,
-                                property_.choices.values())
+      dst_dict['choices'] = [self._GenerateProperty(c)
+                             for c in property_.choices.values()]
       # We keep track of which is last for knowing when to add "or" between
       # choices in templates.
       if len(dst_dict['choices']) > 0:
         dst_dict['choices'][-1]['last'] = True
     elif property_.type_ == model.PropertyType.REF:
-      dst_dict['link'] = GetLinkToRefType(self._namespace.name,
-                                          property_.ref_type)
+      dst_dict['link'] = self._GetLink(property_.ref_type)
     elif property_.type_ == model.PropertyType.ARRAY:
       dst_dict['array'] = self._GenerateProperty(property_.item_type)
     elif property_.type_ == model.PropertyType.ENUM:
       dst_dict['enum_values'] = []
       for enum_value in property_.enum_values:
         dst_dict['enum_values'].append({'name': enum_value})
       if len(dst_dict['enum_values']) > 0:
         dst_dict['enum_values'][-1]['last'] = True
     elif property_.instance_of is not None:
       dst_dict['simple_type'] = property_.instance_of.lower()
     else:
       dst_dict['simple_type'] = property_.type_.name.lower()
 
 class _LazySamplesGetter(object):
   """This class is needed so that an extensions API page does not have to fetch
   the apps samples page and vice versa.
   """
   def __init__(self, api_name, samples):
     self._api_name = api_name
     self._samples = samples
 
   def get(self, key):
     return self._samples.FilterSamples(key, self._api_name)
 
 class APIDataSource(object):
   """This class fetches and loads JSON APIs from the FileSystem passed in with
   |cache_factory|, so the APIs can be plugged into templates.
   """
   class Factory(object):
-    def __init__(self, cache_factory, base_path, samples_factory):
+    def __init__(self,
+                 cache_factory,
+                 base_path):
       self._permissions_cache = cache_factory.Create(self._LoadPermissions,
                                                      compiled_fs.PERMS,
                                                      version=_VERSION)
-      self._json_cache = cache_factory.Create(self._LoadJsonAPI,
-                                              compiled_fs.JSON,
-                                              version=_VERSION)
-      self._idl_cache = cache_factory.Create(self._LoadIdlAPI,
-                                             compiled_fs.IDL,
-                                             version=_VERSION)
+      self._json_cache = cache_factory.Create(
+          lambda api: self._LoadJsonAPI(api, False),
+          compiled_fs.JSON,
+          version=_VERSION)
+      self._idl_cache = cache_factory.Create(
+          lambda api: self._LoadIdlAPI(api, False),
+          compiled_fs.IDL,
+          version=_VERSION)
+      self._json_cache_no_refs = cache_factory.Create(

[not at google - send to devlin, 2012/11/05 19:47:50]

Nice. Quick comment on why having separate caches with refs vs no refs is
necessary?

[cduvall, 2012/11/06 00:58:54]

Done.

+          lambda api: self._LoadJsonAPI(api, True),
+          compiled_fs.JSON_NO_REFS,
+          version=_VERSION)
+      self._idl_cache_no_refs = cache_factory.Create(
+          lambda api: self._LoadIdlAPI(api, True),
+          compiled_fs.IDL_NO_REFS,
+          version=_VERSION)
       self._idl_names_cache = cache_factory.Create(self._GetIDLNames,
                                                    compiled_fs.IDL_NAMES,
                                                    version=_VERSION)
-      self._samples_factory = samples_factory
+      self._names_cache = cache_factory.Create(self._GetAllNames,
+                                               compiled_fs.NAMES,
+                                               version=_VERSION)
       self._base_path = base_path
+      self._ref_resolver_factory = None

[not at google - send to devlin, 2012/11/05 19:47:50]

# These must be set later via the SetFooDataSourceFactory methods.

[cduvall, 2012/11/06 00:58:54]

Done.

+      self._samples_data_source_factory = None
 
-    def Create(self, request):
+    def SetSamplesDataSourceFactory(self, samples_data_source_factory):
+      self._samples_data_source_factory = samples_data_source_factory
+
+    def SetReferenceResolverFactory(self, ref_resolver_factory):
+      self._ref_resolver_factory = ref_resolver_factory
+
+    def Create(self, request, disable_refs=False):
+      """Create an APIDataSource. |disable_refs| specifies whether $ref's in
+      APIs being processed by the |ToDict| method of _JSCModel follows $ref's
+      in the API. This prevents endless recursion in ReferenceResolver.
+      """
+      if self._samples_data_source_factory is None:

[cduvall, 2012/11/03 01:30:05]

I have this chunk here because SamplesDataSource needs a ReferenceResolver, and
ReferenceResolver needs an APIDataSource. So we need to be able to make an
APIDataSource without the factory having a SampelsDataSource.Factory.

The same applies below, with ReferenceResolver and APIDataSource being
interdependent.

+        # Only error if there is a request, which means this APIDataSource is
+        # actually being used to render a page.
+        if request is not None:
+          logging.error('SamplesDataSource.Factory was never set in '
+                        'APIDataSource.Factory.')
+        samples = None
+      else:
+        samples = self._samples_data_source_factory.Create(request)
+      if not disable_refs and self._ref_resolver_factory is None:
+        logging.error('ReferenceResolver.Factory was never set in '
+                      'APIDataSource.Factory.')
       return APIDataSource(self._permissions_cache,
                            self._json_cache,
                            self._idl_cache,
+                           self._json_cache_no_refs,
+                           self._idl_cache_no_refs,
+                           self._names_cache,
                            self._idl_names_cache,
                            self._base_path,
-                           self._samples_factory.Create(request))
+                           samples,
+                           disable_refs)
 
     def _LoadPermissions(self, json_str):
       return json.loads(json_comment_eater.Nom(json_str))
 
-    def _LoadJsonAPI(self, api):
-      return _JscModel(json.loads(json_comment_eater.Nom(api))[0])
+    def _LoadJsonAPI(self, api, disable_refs):
+      return _JSCModel(
+          json.loads(json_comment_eater.Nom(api))[0],
+          self._ref_resolver_factory.Create() if not disable_refs else None,
+          disable_refs).ToDict()
 
-    def _LoadIdlAPI(self, api):
+    def _LoadIdlAPI(self, api, disable_refs):
       idl = idl_parser.IDLParser().ParseData(api)
-      return _JscModel(idl_schema.IDLSchema(idl).process()[0])
+      return _JSCModel(
+          idl_schema.IDLSchema(idl).process()[0],
+          self._ref_resolver_factory.Create() if not disable_refs else None,
+          disable_refs).ToDict()
 
     def _GetIDLNames(self, apis):
-      return [model.UnixName(os.path.splitext(api.split('/')[-1])[0])
-              for api in apis if api.endswith('.idl')]
+      return [
+        model.UnixName(os.path.splitext(api[len('%s/' % self._base_path):])[0])
+        for api in apis if api.endswith('.idl')
+      ]
+
+    def _GetAllNames(self, apis):
+      return [
+        model.UnixName(os.path.splitext(api[len('%s/' % self._base_path):])[0])
+        for api in apis
+      ]
 
   def __init__(self,
                permissions_cache,
                json_cache,
                idl_cache,
+               json_cache_no_refs,
+               idl_cache_no_refs,
+               names_cache,
                idl_names_cache,
                base_path,
-               samples):
+               samples,
+               disable_refs):
     self._base_path = base_path
     self._permissions_cache = permissions_cache
     self._json_cache = json_cache
     self._idl_cache = idl_cache
+    self._json_cache_no_refs = json_cache_no_refs
+    self._idl_cache_no_refs = idl_cache_no_refs
+    self._names_cache = names_cache
     self._idl_names_cache = idl_names_cache
     self._samples = samples
+    self._disable_refs = disable_refs
 
   def _GetPermsFromFile(self, filename):
     try:
       perms = self._permissions_cache.GetFromFile('%s/%s' %
           (self._base_path, filename))
       return dict((model.UnixName(k), v) for k, v in perms.iteritems())
     except FileNotFoundError:
       return {}
 
   def _GetFeature(self, path):
     # Remove 'experimental_' from path name to match the keys in
     # _permissions_features.json.
     path = model.UnixName(path.replace('experimental_', ''))
     for filename in ['_permission_features.json', '_manifest_features.json']:
       api_perms = self._GetPermsFromFile(filename).get(path, None)
       if api_perms is not None:
         break
     if api_perms and api_perms['channel'] in ('trunk', 'dev', 'beta'):
       api_perms[api_perms['channel']] = True
     return api_perms
 
-  def _GenerateHandlebarContext(self, handlebar, path):
+  def _GenerateHandlebarContext(self, handlebar_dict, path):
     return_dict = {
       'permissions': self._GetFeature(path),
       'samples': _LazySamplesGetter(path, self._samples)
     }
-    return_dict.update(handlebar.ToDict())
+    return_dict.update(handlebar_dict)
     return return_dict
 
-  def __getitem__(self, key):
-    return self.get(key)
+  def _GetAsSubdirectory(self, name):
+    if name.startswith('experimental_'):
+      parts = name[len('experimental_'):].split('_', 1)
+      parts[1] = 'experimental_%s' % parts[1]
+      return '/'.join(parts)
+    return name.replace('_', '/', 1)
 
   def get(self, key):
-    path, ext = os.path.splitext(key)
+    if key.endswith('.html') or key.endswith('.json') or key.endswith('.idl'):
+      path, ext = os.path.splitext(key)
+    else:
+      path = key
     unix_name = model.UnixName(path)
     idl_names = self._idl_names_cache.GetFromFileListing(self._base_path)
-    cache, ext = ((self._idl_cache, '.idl') if (unix_name in idl_names) else
-                  (self._json_cache, '.json'))
+    names = self._names_cache.GetFromFileListing(self._base_path)
+    if unix_name not in names and self._GetAsSubdirectory(unix_name) in names:
+      unix_name = self._GetAsSubdirectory(unix_name)
+
+    if self._disable_refs:
+      cache, ext = (
+          (self._idl_cache_no_refs, '.idl') if (unix_name in idl_names) else
+          (self._json_cache_no_refs, '.json'))
+    else:
+      cache, ext = ((self._idl_cache, '.idl') if (unix_name in idl_names) else
+                    (self._json_cache, '.json'))
     return self._GenerateHandlebarContext(
         cache.GetFromFile('%s/%s%s' % (self._base_path, unix_name, ext)),
         path)