Dynamically generate a heading for Extension Docs API pages

chrome/common/extensions/docs/server2/availability_data_source.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import logging
+import os
+
+from appengine_url_fetcher import AppEngineUrlFetcher
+from caching_file_system import CachingFileSystem
+from compiled_file_system import CompiledFileSystem
+from subversion_file_system import SubversionFileSystem
+import svn_constants
+from third_party.json_schema_compiler import json_comment_eater
+from third_party.json_schema_compiler import model
+import url_constants
+
+# These labels are used for interacting with AvailabilityDataSource's object
+# store. The format is <label><string> (e.g. svn21, or permissiontabs26)
+SVN = 'svn'
+PERMISSION = 'permission'
+MANIFEST = 'manifest'
+FILESYS = 'filesys'

[not at google - send to devlin, 2013/04/27 01:48:34]

this is what the category= parameter to ObjectStoreCreator.Create is for, it
does namespacing automatically.

+
+class AvailabilityDataSource(object):
+  '''Uses API data sources generated by a ChromeVersionDataSource in order to
+  search the filesystem for the earliest existence of a specified API throughout
+  the different versions of Chrome; this constitutes an API's availability.
+  '''
+  class Factory(object):
+    def __init__(self,
+                 chrome_version_utility,
+                 object_store_creator_factory,
+                 svn_file_system):
+      self._chrome_version_utility = chrome_version_utility
+      self._object_store_creator_factory = object_store_creator_factory
+      json_string = svn_file_system.ReadSingle('%s/api_availabilities.json'
+                                               % svn_constants.JSON_PATH)
+      self._existing_availabilities = json.loads(json_string)
+      json_string = svn_file_system.ReadSingle('%s/_permission_features.json'
+                                               % svn_constants.API_PATH)
+      self._permission_features = json.loads(json_comment_eater.Nom(
+          json_string))
+      json_string = svn_file_system.ReadSingle('%s/_manifest_features.json'
+                                               % svn_constants.API_PATH)
+      self._manifest_features = json.loads(json_comment_eater.Nom(json_string))
+
+    def Create(self):
+      return AvailabilityDataSource(self._chrome_version_utility,
+                                    self._object_store_creator_factory,
+                                    self._existing_availabilities,
+                                    self._permission_features,
+                                    self._manifest_features)
+
+  def __init__(self,
+               chrome_version_utility,
+               object_store_creator_factory,
+               existing_availabilities,
+               permission_features,
+               manifest_features):
+    self._chrome_version_utility = chrome_version_utility
+    self._object_store_creator_factory = object_store_creator_factory
+    self._object_store = self._object_store_creator_factory.Create(
+        AvailabilityDataSource).Create()
+    self._existing_availabilities = existing_availabilities
+    self._permission_features = permission_features
+    self._manifest_features = manifest_features
+    self._permission_apis = []
+    self._manifest_apis = []
+    self._orphan_apis = []
+
+  def _CreateSvnFileSystemForBranch(self, branch):
+    '''Creates a file system tied to a specific |branch| of the svn repository.
+    '''
+    if branch == 'trunk':
+      svn_url = '/'.join((url_constants.SVN_TRUNK_URL,
+                          'src',
+                          svn_constants.EXTENSIONS_PATH))
+    else:
+      svn_url = '/'.join((url_constants.SVN_BRANCH_URL,
+                          branch,
+                          'src',
+                          svn_constants.EXTENSIONS_PATH))
+    viewvc_url = svn_url.replace(url_constants.SVN_URL,
+                                 url_constants.VIEWVC_URL)
+
+    svn_file_system = CachingFileSystem(
+        SubversionFileSystem(AppEngineUrlFetcher(svn_url),
+                             AppEngineUrlFetcher(viewvc_url)),
+        self._object_store_creator_factory)

[not at google - send to devlin, 2013/04/27 01:48:34]

so this is obviously the same as server instance's version, and we should re-use
the code.

in fact we need to re-use the code, since part of the contract of all this stuff
now is that appengine instances aren't allowed to touch the network, only object
store - so this class needs to know to construct offline subversion file systems
in that case, and online file systems for cron jobs.

you'll probably need to inject a factory-style method/class into here that can
construct SubversionFileSystems from a branch - or at least, one that's either
wrapped in a cache or an offline version.

would it work for those methods to actually just create a whole ServerInstance
then access the svn_file_sytsem from it? Like

--- server_instance.py

@staticmethod
def CreateOnline(channel):
  @memoize
  def svn_file_system_creator(branch):
    return CreateOnline(branch).svn_file_system   <-- oh crap it's a branch not
a channel
  return ServerInstance(..., svn_file_system_creator)

@staticmethod
@memoize
def CreateOffline(channel):
  def svn_file_system_creator(branch):
    return CreateOffline(branch).svn_file_system  <-- same oh crap
  return ServerInstance(..., svn_file_system_creator)

+    return svn_file_system
+
+  def _GetChannelFromPermissionFeatures(self, api_name, permission_features):
+    '''Sometimes the API information will be in a dict, other times it will be
+    in a dict contained in a list.
+    '''
+    api_info = permission_features.get(api_name, None)
+    if api_info is not None:
+      try:
+        channel = api_info.get('channel')
+      except AttributeError:
+        channel = api_info[0].get('channel')
+      finally:
+        return channel
+    return None
+
+  def _CheckStablePermissionExistence(self, api_name, version):
+    '''Checks _permission_features.json for the given |version| of Chrome,
+    and returns a boolean representing whether or not the API was available on
+    the stable channel in |version|.
+    '''
+    existence = self._object_store.Get('%s%s%s'
+                                       % (PERMISSION, api_name, version)).Get()
+    if existence is not None:
+      return existence
+
+    svn_file_system = self._object_store.Get('%s%s' % (SVN, version)).Get()
+    if svn_file_system is None:
+      svn_file_system = self._CreateSvnFileSystemForBranch(
+          self._chrome_version_utility.GetBranchNumberForVersion(version))
+      self._object_store.Set('%s%s' % (SVN, version), svn_file_system)
+
+    json_string = svn_file_system.ReadSingle('%s/_permission_features.json'
+                                             % svn_constants.API_PATH)
+    permission_features = json.loads(json_comment_eater.Nom(json_string))
+
+    existence = 'stable' == self._GetChannelFromPermissionFeatures(
+        api_name, permission_features)
+    self._object_store.Set('%s%s%s' % (PERMISSION, api_name, version),
+                           existence)
+    return existence
+
+  def _CheckManifestExistence(self, api_name, version):
+    '''As a fallback measure for an API not being found in _permission_features,
+    check _manifest_features.json for the given |version| of Chrome,
+    and return a boolean representing whether or not the API was represented
+    within the file.
+    '''
+    #_manifest_features.json uses unix_hacker_style API names.
+    api_name = model.UnixName(api_name)
+    existence = self._object_store.Get('%s%s%s'
+                                       % (MANIFEST, api_name, version)).Get()
+    if existence is not None:
+      return existence
+
+    svn_file_system = self._object_store.Get('%s%s' % (SVN, version)).Get()
+    if svn_file_system is None:
+      svn_file_system = self._CreateSvnFileSystemForBranch(
+          self._chrome_version_utility.GetBranchNumberForVersion(version))
+      self._object_store.Set('%s%s' % (SVN, version), svn_file_system)
+
+    json_string = svn_file_system.ReadSingle('%s/_manifest_features.json'
+                                             % svn_constants.API_PATH)
+    manifest_features = json.loads(json_comment_eater.Nom(json_string))
+    if manifest_features.get(api_name, None) is not None:
+      existence = True
+    else:
+      existence = False
+    self._object_store.Set('%s%s%s' % (MANIFEST, api_name, version), existence)
+    return existence
+
+  def _GetAllNames(self, files_dict):
+    '''Returns extension-less names for a list of files.
+    '''
+    files = files_dict[svn_constants.API_PATH + '/']
+    return [model.UnixName(os.path.splitext(f)[0]) for f in files]
+
+  def _CheckFileSystemExistence(self, api_name, version):
+    '''Uses a list of API names for a given branch to check simply for the
+    existence of a given API in that branch.
+    '''
+    names = self._object_store.Get('%s%s' % (FILESYS, version)).Get()
+    if names is not None:
+      return api_name in names
+
+    svn_file_system = self._object_store.Get('%s%s' % (SVN, version)).Get()
+    if svn_file_system is None:
+      svn_file_system = self._CreateSvnFileSystemForBranch(
+          self._chrome_version_utility.GetBranchNumberForVersion(version))
+      self._object_store.Set('%s%s' % (SVN, version), svn_file_system)
+    names = svn_file_system.Read([svn_constants.API_PATH + '/']).Get()
+    names = self._GetAllNames(names)
+    self._object_store.Set('%s%s' % (FILESYS, version), names)
+    return api_name in names
+
+  def _FindEarliestStableAvailability(self, api_name, version):
+    '''Searches in descending order through filesystem caches tied to specific
+    chrome version numbers and looks for the availability of an API, |api_name|,
+    on the stable channel. When a version is found where the API is no longer
+    available on stable, returns the previous version number (the last known
+    version where the API was stable).
+    '''
+    # The _permission_features.json file (with 'channel' keys) is present only
+    # in Chrome 20 and onwards.
+    while version >= 20:
+      if ((api_name in self._permission_apis
+          and self._CheckStablePermissionExistence(api_name, str(version)))
+        or (api_name in self._manifest_apis
+            and self._CheckManifestExistence(api_name, str(version)))
+        or (api_name in self._orphan_apis
+            and self._CheckFileSystemExistence(api_name, str(version)))):
+        version -= 1
+      else:
+        break
+    version += 1
+    return str(version)
+
+  def GetAvailability(self, api_name):
+    '''Determines the availability for an API by testing several scenarios.
+    (i.e. Is the API experimental? Only available on certain development
+    channels? If it's stable, when did it first become stable? etc.)
+    '''
+    availability = self._object_store.Get(api_name).Get()
+    if availability is not None:
+      return availability
+
+    # Check for a predetermined availability for this API.
+    availability = self._existing_availabilities.get(api_name, None)
+    if availability is not None:
+      self._object_store.Set(api_name, availability)
+      return availability
+
+    latest_version = str(self._chrome_version_utility.GetLatestVersionNumber())
+    channel = self._GetChannelFromPermissionFeatures(api_name,
+                                                     self._permission_features)
+    if channel is not None:
+       # This is a special case: availability can be any of the dev channels,
+       # to check for this information instead of checking just for existence.
+       availability = channel
+       self._permission_apis.append(api_name)
+    elif self._CheckManifestExistence(api_name, latest_version):
+      availability = 'stable'
+      self._manifest_apis.append(api_name)
+    elif self._CheckFileSystemExistence(api_name, latest_version):
+      availability = 'stable'
+      self._orphan_apis.append(api_name)
+    else:
+      logging.warning('%s could not be located. %s\'s availability is None.'
+                      % (api_name, api_name))
+      availability = None
+
+    if availability != 'stable':
+      self._object_store.Set(api_name, availability)
+      return availability
+
+    availability = self._FindEarliestStableAvailability(api_name,
+                                                        int(latest_version) - 1)
+    self._object_store.Set(api_name, availability)
+    return availability