Extensions Docs Server: Generalize $ref's to work for any schema node

chrome/common/extensions/docs/server2/api_data_source.py

Index: chrome/common/extensions/docs/server2/api_data_source.py
diff --git a/chrome/common/extensions/docs/server2/api_data_source.py b/chrome/common/extensions/docs/server2/api_data_source.py
index 49421a60f7dce381f57afb087093c8fe099cf006..70c6f12bd19a26e8dbeabf4797fdf767f0b3a734 100644
--- a/chrome/common/extensions/docs/server2/api_data_source.py
+++ b/chrome/common/extensions/docs/server2/api_data_source.py
@@ -4,6 +4,7 @@
 
 import copy
 import json
+import logging
 import os
 
 from docs_server_utils import GetLinkToRefType
@@ -17,7 +18,7 @@ import third_party.json_schema_compiler.idl_parser as idl_parser
 # Increment this version when there are changes to the data stored in any of
 # the caches used by APIDataSource. This allows the cache to be invalidated
 # without having to flush memcache on the production server.
-_VERSION = 2
+_VERSION = 3
 
 def _RemoveNoDocs(item):
   if type(item) == dict:
@@ -49,19 +50,35 @@ def _FormatValue(value):
   s = str(value)
   return ','.join([s[max(0, i - 3):i] for i in range(len(s), 0, -3)][::-1])
 
-class _JscModel(object):
+class _JSCModel(object):
   """Uses a Model from the JSON Schema Compiler and generates a dict that
   a Handlebar template can use for a data source.
   """
-  def __init__(self, json):
+  def __init__(self, json, api_data_source, api_list_data_source):
+    self._api_data_source = api_data_source
+    self._api_list = api_list_data_source.GetAllNames()

[not at google - send to devlin, 2012/11/01 22:00:49]

These JSCModel objects are pickled, right? That's what we've made the cache
factories do. So, it seems problematic to me that we're assigning these as
member variables.

It's a bit yuck, but perhaps passing them through ToDict is the only way to go
for now.

[cduvall, 2012/11/02 01:53:53]

Done.

     clean_json = copy.deepcopy(json)
     if _RemoveNoDocs(clean_json):
       self._namespace = None
     else:
       self._namespace = model.Namespace(clean_json, clean_json['namespace'])
 
-  def _FormatDescription(self, description):
-    if description is None or '$ref:' not in description:
+  def _GetLinkToRefType(self, ref):
+    ref_data = GetLinkToRefType(self._namespace.name,
+                                ref,
+                                self._api_data_source,
+                                self._api_list)
+    if ref_data is None:
+      logging.error('$ref %s could not be resolved.' % ref)
+      if '.' in ref:
+        type_name = ref.rsplit('.', 1)[-1]
+      else:
+        type_name = ref
+      return { 'href': '#type-%s' % type_name, 'text': ref }
+    return ref_data
+
+  def _FormatDescription(self, description, resolve_refs):
+    if description is None or '$ref:' not in description or not resolve_refs:
       return description
     refs = description.split('$ref:')
     formatted_description = [refs[0]]
@@ -76,43 +93,47 @@ class _JscModel(object):
       if not ref[-1].isalnum():
         rest = ref[-1] + rest
         ref = ref[:-1]
-      ref_dict = GetLinkToRefType(self._namespace.name, ref)
+      ref_dict = self._GetLinkToRefType(ref)
       formatted_description.append('<a href="%(href)s">%(text)s</a>%(rest)s' %
           { 'href': ref_dict['href'], 'text': ref_dict['text'], 'rest': rest })
     return ''.join(formatted_description)
 
-  def ToDict(self):
+  def ToDict(self, resolve_refs):
     if self._namespace is None:
       return {}
     return {
       'name': self._namespace.name,
-      'types':  [self._GenerateType(t) for t in self._namespace.types.values()
+      'types':  [self._GenerateType(t, resolve_refs)
+                 for t in self._namespace.types.values()
                  if t.type_ != model.PropertyType.ADDITIONAL_PROPERTIES],
-      'functions': self._GenerateFunctions(self._namespace.functions),
-      'events': self._GenerateEvents(self._namespace.events),
-      'properties': self._GenerateProperties(self._namespace.properties)
+      'functions': self._GenerateFunctions(self._namespace.functions,
+                                           resolve_refs),
+      'events': self._GenerateEvents(self._namespace.events, resolve_refs),
+      'properties': self._GenerateProperties(self._namespace.properties,
+                                             resolve_refs)
     }
 
-  def _GenerateType(self, type_):
+  def _GenerateType(self, type_, resolve_refs):
     type_dict = {
       'name': type_.simple_name,
-      'description': self._FormatDescription(type_.description),
-      'properties': self._GenerateProperties(type_.properties),
-      'functions': self._GenerateFunctions(type_.functions),
-      'events': self._GenerateEvents(type_.events),
+      'description': self._FormatDescription(type_.description, resolve_refs),
+      'properties': self._GenerateProperties(type_.properties, resolve_refs),
+      'functions': self._GenerateFunctions(type_.functions, resolve_refs),
+      'events': self._GenerateEvents(type_.events, resolve_refs),
       'id': _CreateId(type_, 'type')
     }
-    self._RenderTypeInformation(type_, type_dict)
+    self._RenderTypeInformation(type_, type_dict, resolve_refs)
     return type_dict
 
-  def _GenerateFunctions(self, functions):
-    return [self._GenerateFunction(f) for f in functions.values()]
+  def _GenerateFunctions(self, functions, resolve_refs):
+    return [self._GenerateFunction(f, resolve_refs) for f in functions.values()]
 
-  def _GenerateFunction(self, function):
+  def _GenerateFunction(self, function, resolve_refs):
     function_dict = {
       'name': function.simple_name,
-      'description': self._FormatDescription(function.description),
-      'callback': self._GenerateCallback(function.callback),
+      'description': self._FormatDescription(function.description,
+                                             resolve_refs),
+      'callback': self._GenerateCallback(function.callback, resolve_refs),
       'parameters': [],
       'returns': None,
       'id': _CreateId(function, 'method')
@@ -123,32 +144,38 @@ class _JscModel(object):
     else:
       function_dict['parent_name'] = None
     if function.returns:
-      function_dict['returns'] = self._GenerateProperty(function.returns)
+      function_dict['returns'] = self._GenerateProperty(function.returns,
+                                                        resolve_refs)
     for param in function.params:
-      function_dict['parameters'].append(self._GenerateProperty(param))
+      function_dict['parameters'].append(self._GenerateProperty(param,
+                                                                resolve_refs))
     if function_dict['callback']:
       function_dict['parameters'].append(function_dict['callback'])
     if len(function_dict['parameters']) > 0:
       function_dict['parameters'][-1]['last'] = True
     return function_dict
 
-  def _GenerateEvents(self, events):
-    return [self._GenerateEvent(e) for e in events.values()]
+  def _GenerateEvents(self, events, resolve_refs):
+    return [self._GenerateEvent(e, resolve_refs) for e in events.values()]
 
-  def _GenerateEvent(self, event):
+  def _GenerateEvent(self, event, resolve_refs):
     event_dict = {
       'name': event.simple_name,
-      'description': self._FormatDescription(event.description),
-      'parameters': map(self._GenerateProperty, event.params),
-      'callback': self._GenerateCallback(event.callback),
-      'conditions': [GetLinkToRefType(self._namespace.name, c)
-                     for c in event.conditions],
-      'actions': [GetLinkToRefType(self._namespace.name, a)
-                     for a in event.actions],
-      'filters': map(self._GenerateProperty, event.filters),
+      'description': self._FormatDescription(event.description,
+                                             resolve_refs),
+      'parameters': [self._GenerateProperty(p, resolve_refs)
+                     for p in event.params],
+      'callback': self._GenerateCallback(event.callback, resolve_refs),
+      'filters': [self._GenerateProperty(f, resolve_refs)
+                  for f in event.filters],
       'supportsRules': event.supports_rules,
       'id': _CreateId(event, 'event')
     }
+    if resolve_refs:
+      event_dict['conditions'] = [self._GetLinkToRefType(condition)
+                                  for condition in event.conditions]
+      event_dict['actions'] = [self._GetLinkToRefType(action)
+                               for action in event.actions]
     if (event.parent is not None and
         not isinstance(event.parent, model.Namespace)):
       event_dict['parent_name'] = event.parent.simple_name
@@ -160,39 +187,44 @@ class _JscModel(object):
       event_dict['parameters'][-1]['last'] = True
     return event_dict
 
-  def _GenerateCallback(self, callback):
+  def _GenerateCallback(self, callback, resolve_refs):
     if not callback:
       return None
     callback_dict = {
       'name': callback.simple_name,
-      'description': self._FormatDescription(callback.description),
+      'description': self._FormatDescription(callback.description,
+                                             resolve_refs),
       'simple_type': {'simple_type': 'function'},
       'optional': callback.optional,
       'parameters': []
     }
     for param in callback.params:
-      callback_dict['parameters'].append(self._GenerateProperty(param))
+      callback_dict['parameters'].append(
+          self._GenerateProperty(param, resolve_refs))
     if (len(callback_dict['parameters']) > 0):
       callback_dict['parameters'][-1]['last'] = True
     return callback_dict
 
-  def _GenerateProperties(self, properties):
-    return [self._GenerateProperty(v) for v in properties.values()
+  def _GenerateProperties(self, properties, resolve_refs):
+    return [self._GenerateProperty(v, resolve_refs) for v in properties.values()
             if v.type_ != model.PropertyType.ADDITIONAL_PROPERTIES]
 
-  def _GenerateProperty(self, property_):
+  def _GenerateProperty(self, property_, resolve_refs):
     property_dict = {
       'name': property_.simple_name,
       'optional': property_.optional,
-      'description': self._FormatDescription(property_.description),
-      'properties': self._GenerateProperties(property_.properties),
+      'description': self._FormatDescription(property_.description,
+                                             resolve_refs),
+      'properties': self._GenerateProperties(property_.properties,
+                                             resolve_refs),
+      'functions': self._GenerateFunctions(property_.functions, resolve_refs),
       'parameters': [],
-      'functions': self._GenerateFunctions(property_.functions),
       'returns': None,
       'id': _CreateId(property_, 'property')
     }
     for param in property_.params:
-      property_dict['parameters'].append(self._GenerateProperty(param))
+      property_dict['parameters'].append(self._GenerateProperty(param,
+                                                                resolve_refs))
     if property_.returns:
       property_dict['returns'] = self._GenerateProperty(property_.returns)
     if (property_.parent is not None and
@@ -206,22 +238,23 @@ class _JscModel(object):
       else:
         property_dict['value'] = property_.value
     else:
-      self._RenderTypeInformation(property_, property_dict)
+      self._RenderTypeInformation(property_, property_dict, resolve_refs)
     return property_dict
 
-  def _RenderTypeInformation(self, property_, dst_dict):
+  def _RenderTypeInformation(self, property_, dst_dict, resolve_refs):
     if property_.type_ == model.PropertyType.CHOICES:
-      dst_dict['choices'] = map(self._GenerateProperty,
-                                property_.choices.values())
+      dst_dict['choices'] = [self._GenerateProperty(c, resolve_refs)
+                             for c in property_.choices.values()]
       # We keep track of which is last for knowing when to add "or" between
       # choices in templates.
       if len(dst_dict['choices']) > 0:
         dst_dict['choices'][-1]['last'] = True
     elif property_.type_ == model.PropertyType.REF:
-      dst_dict['link'] = GetLinkToRefType(self._namespace.name,
-                                          property_.ref_type)
+      if resolve_refs:
+        dst_dict['link'] = self._GetLinkToRefType(property_.ref_type)
     elif property_.type_ == model.PropertyType.ARRAY:
-      dst_dict['array'] = self._GenerateProperty(property_.item_type)
+      dst_dict['array'] = self._GenerateProperty(property_.item_type,
+                                                 resolve_refs)
     elif property_.type_ == model.PropertyType.ENUM:
       dst_dict['enum_values'] = []
       for enum_value in property_.enum_values:
@@ -249,7 +282,10 @@ class APIDataSource(object):
   |cache_factory|, so the APIs can be plugged into templates.
   """
   class Factory(object):
-    def __init__(self, cache_factory, base_path, samples_factory):
+    def __init__(self,
+                 cache_factory,
+                 base_path,
+                 api_list_data_source):
       self._permissions_cache = cache_factory.Create(self._LoadPermissions,
                                                      compiled_fs.PERMS,
                                                      version=_VERSION)
@@ -262,44 +298,66 @@ class APIDataSource(object):
       self._idl_names_cache = cache_factory.Create(self._GetIDLNames,
                                                    compiled_fs.IDL_NAMES,
                                                    version=_VERSION)
-      self._samples_factory = samples_factory
+      self._names_cache = cache_factory.Create(self._GetAllNames,
+                                               compiled_fs.NAMES,
+                                               version=_VERSION)
       self._base_path = base_path
+      self._api_list_data_source = api_list_data_source
+      self._api_data_source = self.Create(None, None, resolve_refs=False)
 
-    def Create(self, request):
+    def Create(self, request, samples_data_source, resolve_refs=True):
       return APIDataSource(self._permissions_cache,
                            self._json_cache,
                            self._idl_cache,
+                           self._names_cache,
                            self._idl_names_cache,
                            self._base_path,
-                           self._samples_factory.Create(request))
+                           samples_data_source,
+                           resolve_refs)
 
     def _LoadPermissions(self, json_str):
       return json.loads(json_comment_eater.Nom(json_str))
 
     def _LoadJsonAPI(self, api):
-      return _JscModel(json.loads(json_comment_eater.Nom(api))[0])
+      return _JSCModel(json.loads(json_comment_eater.Nom(api))[0],
+                       self._api_data_source,
+                       self._api_list_data_source)
 
     def _LoadIdlAPI(self, api):
       idl = idl_parser.IDLParser().ParseData(api)
-      return _JscModel(idl_schema.IDLSchema(idl).process()[0])
+      return _JSCModel(idl_schema.IDLSchema(idl).process()[0],
+                       self._api_data_source,
+                       self._api_list_data_source)
 
     def _GetIDLNames(self, apis):
-      return [model.UnixName(os.path.splitext(api.split('/')[-1])[0])
-              for api in apis if api.endswith('.idl')]
+      return [
+        model.UnixName(os.path.splitext(api[len('%s/' % self._base_path):])[0])
+        for api in apis if api.endswith('.idl')
+      ]
+
+    def _GetAllNames(self, apis):
+      return [
+        model.UnixName(os.path.splitext(api[len('%s/' % self._base_path):])[0])
+        for api in apis
+      ]
 
   def __init__(self,
                permissions_cache,
                json_cache,
                idl_cache,
+               names_cache,
                idl_names_cache,
                base_path,
-               samples):
+               samples,
+               resolve_refs):
     self._base_path = base_path
     self._permissions_cache = permissions_cache
     self._json_cache = json_cache
     self._idl_cache = idl_cache
+    self._names_cache = names_cache
     self._idl_names_cache = idl_names_cache
     self._samples = samples
+    self._resolve_refs = resolve_refs
 
   def _GetPermsFromFile(self, filename):
     try:
@@ -326,16 +384,30 @@ class APIDataSource(object):
       'permissions': self._GetFeature(path),
       'samples': _LazySamplesGetter(path, self._samples)
     }
-    return_dict.update(handlebar.ToDict())
+    return_dict.update(handlebar.ToDict(resolve_refs=self._resolve_refs))

[not at google - send to devlin, 2012/11/01 22:00:49]

Perhaps you'll need to explain to me what all this resolve_refs stuff is.

[cduvall, 2012/11/02 01:53:53]

Done, explained in the Create method above.

     return return_dict
 
+  def _GetAsSubdirectory(self, name):
+    if name.startswith('experimental_'):
+      parts = name[len('experimental_'):].split('_', 1)
+      parts[1] = 'experimental_%s' % parts[1]
+      return '/'.join(parts)
+    return name.replace('_', '/', 1)
+
   def __getitem__(self, key):
     return self.get(key)
 
   def get(self, key):
-    path, ext = os.path.splitext(key)
+    if key.endswith('.html') or key.endswith('.json') or key.endswith('.idl'):
+      path, ext = os.path.splitext(key)
+    else:
+      path = key
     unix_name = model.UnixName(path)
     idl_names = self._idl_names_cache.GetFromFileListing(self._base_path)
+    names = self._names_cache.GetFromFileListing(self._base_path)
+    if unix_name not in names and self._GetAsSubdirectory(unix_name) in names:
+      unix_name = self._GetAsSubdirectory(unix_name)
+
     cache, ext = ((self._idl_cache, '.idl') if (unix_name in idl_names) else
                   (self._json_cache, '.json'))
     return self._GenerateHandlebarContext(