Dynamically generate a heading for Extension Docs API pages

chrome/common/extensions/docs/server2/availability_finder.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import os
+
+from branch_utility import BranchUtility
+from chrome_version_utility import ChromeVersionUtility
+from compiled_file_system import CompiledFileSystem
+from file_system import FileNotFoundError
+import svn_constants
+from third_party.json_schema_compiler import json_parse, model
+
+API_AVAILABILITIES = 'api_availabilities.json'
+EXTENSION_API = 'extension_api.json'
+MANIFEST_FEATURES = '_manifest_features.json'
+PERMISSION_FEATURES = '_permission_features.json'

[not at google - send to devlin, 2013/05/15 08:15:00]

these are all private so should start with a _, and maybe you should join them
with their prefixes (JSON_PATH, API_PATH, whatever) up-front to make that code
smaller + do less work.

[epeterson, 2013/06/02 00:25:50]

Done.

+
+def _GetChannelFromPermissionFeature(permission_feature):
+  '''Handles finding API channel information from _permission_features.json.
+  Sometimes this info will be in a dict, but other times it will be
+  in a dict contained in a list.
+  '''
+  channel = None
+  if permission_feature is not None:
+    if isinstance(permission_feature, collections.Mapping):
+      channel = permission_feature.get('channel')
+    else:
+      channel = BranchUtility.NewestChannel(entry.get('channel')
+                                             for entry in permission_feature)
+  return channel
+
+class AvailabilityFinder(object):
+  '''Uses API data sources generated by a ChromeVersionDataSource in order to
+  search the filesystem for the earliest existence of a specified API throughout
+  the different versions of Chrome; this constitutes an API's availability.
+  '''
+  class AvailabilityInfo(object):

[not at google - send to devlin, 2013/05/15 08:15:00]

this object is picklable, so it can be directly put into the object stores. you
don't need to convert it to and from tuples yourself.

[epeterson, 2013/06/02 00:25:50]

Done. (I had tried this initially, but I think it must not've worked due to
being an inner class)

+    def __init__(self, channel, version):
+      self.channel = channel
+      self.version = version
+
+  class Factory(object):
+    def __init__(self,
+                 chrome_version_utility,
+                 object_store_creator,
+                 compiled_host_fs_factory,
+                 create_file_system):
+      self._chrome_version_utility = chrome_version_utility
+      self._object_store_creator = object_store_creator
+      self._compiled_host_fs_factory = compiled_host_fs_factory
+      self._create_file_system = create_file_system
+
+    def Create(self):
+      return AvailabilityFinder(self._chrome_version_utility,
+                                self._object_store_creator,
+                                self._compiled_host_fs_factory,
+                                self._create_file_system)
+
+  def __init__(self,
+               chrome_version_utility,
+               object_store_creator,
+               compiled_host_fs_factory,
+               create_file_system):
+    self._chrome_version_utility = chrome_version_utility
+    self._object_store_creator = object_store_creator
+    self._json_cache = compiled_host_fs_factory.Create(self._ParseForCompiledFS,

[not at google - send to devlin, 2013/05/15 08:15:00]

just pass in json_parse.Parse, no need to wrap it in another function

[epeterson, 2013/06/02 00:25:50]

compiled filesystems send a path and file data to their populate function, but
json_parse.Parse only accepts one paramater, a json string. So, using lambdas
here to compensate. 

+                                                       AvailabilityFinder,
+                                                       'json-cache')
+    self._branch_utility = BranchUtility.Create(object_store_creator)
+    self._create_file_system = create_file_system
+    self._object_store = object_store_creator.Create(AvailabilityFinder)
+    self._permission_apis = []

[not at google - send to devlin, 2013/05/15 08:15:00]

as I mentioned in IRC - holding onto these
(permission_apis,manifest_apis,orphan_apis) probably doesn't gain much since
you'll be caching the result of GetApiAvailability anyway, and will only end up
reading them once in the common case.

[epeterson, 2013/06/02 00:25:50]

The intent was to use these to force APIs that were, for instance, found in the
_manifest_features.json file in trunk  after falling back from
_permission_features, to only be checked in _manifest_features for the other
versions.
APIs can shift around between the *_features.json files, so I thought that
needed to be restricted. 

Based on your comments on IRC, however, it sounds like it's fine if the APIs
move around between _features files, so they don't need to be divided up into
subsets like this.

+    self._manifest_apis = []
+    self._orphan_apis = []
+
+  def _ParseForCompiledFS(self, path, json):
+    '''Wrapper method to be passed to compiled_file_systems upon creation.
+    '''
+    return json_parse.Parse(json)
+
+  def _CreateCompiledFS(self, branch):
+    # TODO(epeterson) Move create_file_system logic into ChromeVersionUtility.

[not at google - send to devlin, 2013/05/15 08:15:00]

don't worry about it

+    compiled_fs_factory = CompiledFileSystem.Factory(
+        self._create_file_system(branch),
+        self._object_store_creator)
+    return compiled_fs_factory.Create(self._ParseForCompiledFS,

[not at google - send to devlin, 2013/05/15 08:15:00]

ditto

[epeterson, 2013/06/02 00:25:50]

Done.

+                                      AvailabilityFinder,
+                                      'branch')
+
+  def _GetExistingAvailability(self, api_name, compiled_fs):
+     api_info = compiled_fs.GetFromFile(
+         '%s/%s' % (svn_constants.JSON_PATH, API_AVAILABILITIES)).get(api_name)
+     if api_info is not None:
+       return [api_info.get('channel'), api_info.get('version')]
+     return None

[not at google - send to devlin, 2013/05/15 08:15:00]

as I mentioned, AvailabilityFinder is picklable, so this can just return it
directly. But since this function is only called in once place, simpler to
inline it.

[epeterson, 2013/06/02 00:25:50]

Done.

+
+  def _GetPermissionFeature(self, api_name, compiled_fs):
+     return compiled_fs.GetFromFile(
+         '%s/%s' % (svn_constants.API_PATH, PERMISSION_FEATURES)).get(api_name)
+
+  def _GetManifestFeature(self, api_name, compiled_fs):
+     return compiled_fs.GetFromFile(
+         '%s/%s' % (svn_constants.API_PATH, MANIFEST_FEATURES)).get(api_name)
+
+  def _GetJSONForExtensionAPI(self, api_name, compiled_fs):
+     return compiled_fs.GetFromFile(
+         '%s/%s' % (svn_constants.API_PATH, EXTENSION_API))
+
+  def _ReadJson(self, svn_file_system, path):

[not at google - send to devlin, 2013/05/15 08:15:00]

not used

[epeterson, 2013/06/02 00:25:50]

Done.

+    return json_parse.Parse(svn_file_system.ReadSingle(path))
+

[not at google - send to devlin, 2013/05/15 08:15:00]

extra \n

[epeterson, 2013/06/02 00:25:50]

Done.

+
+  def _CheckStablePermissionExistence(self, api_name, version):
+    '''Checks _permission_features.json for the given |version| of Chrome,
+    and returns a boolean representing whether or not the API was available on
+    the stable channel in |version|.
+    '''
+    compiled_fs = self._CreateCompiledFS(
+        self._chrome_version_utility.GetBranchNumberForVersion(version))
+    permission_feature = self._GetPermissionFeature(api_name, compiled_fs)
+    existence = 'stable' == _GetChannelFromPermissionFeature(permission_feature)
+    return existence
+
+  def _CheckChannelPermissionExistence(self, api_name, version, channel):
+    compiled_fs = self._CreateCompiledFS(
+        self._chrome_version_utility.GetBranchNumberForVersion(version))
+    permission_feature = self._GetPermissionFeature(api_name, compiled_fs)
+    existence = channel == _GetChannelFromPermissionFeature(permission_feature)
+    return existence
+

[not at google - send to devlin, 2013/05/15 08:15:00]

extra \n

[epeterson, 2013/06/02 00:25:50]

Done.

+
+  def _CheckSimplePermissionExistence(self, api_name, version):

[not at google - send to devlin, 2013/05/15 08:15:00]

looks like this method can be combined with checkstablepermissionexistence, if
they were to take a file system.

And call it _ExistsInPermissionFeatures(api_name, file_system)

[epeterson, 2013/06/02 00:25:50]

Logic has been moved up above.

+    compiled_fs = self._CreateCompiledFS(
+        self._chrome_version_utility.GetBranchNumberForVersion(version))
+    permission_feature = self._GetPermissionFeature(api_name, compiled_fs)
+    return permission_feature is not None
+
+  def _CheckManifestExistence(self, api_name, version):

[not at google - send to devlin, 2013/05/15 08:15:00]

_ExistsInManifestFeatures, take a file system

[epeterson, 2013/06/02 00:25:50]

Done.

+    '''As a fallback measure for an API not being found in _permission_features,
+    check _manifest_features.json for the given |version| of Chrome,
+    and return a boolean representing whether or not the API was represented
+    within the file.
+    '''
+    #_manifest_features.json uses unix_hacker_style API names.
+    api_name = model.UnixName(api_name)
+    compiled_fs = self._CreateCompiledFS(
+        self._chrome_version_utility.GetBranchNumberForVersion(version))
+    manifest_feature = self._GetManifestFeature(api_name, compiled_fs)
+    return manifest_feature is not None
+
+  def _GetAllNames(self, files):
+    '''Returns extension-less names for a list of files.
+    '''
+    return [os.path.splitext(f)[0] for f in files]
+
+  def _CheckFileSystemExistence(self, api_name, version):

[not at google - send to devlin, 2013/05/15 08:15:00]

_ExistsInFileSystem, take a file system

[epeterson, 2013/06/02 00:25:50]

Done.

+    '''Uses a list of API names for a given branch to check simply for the
+    existence of a given API in that branch.
+    '''
+    names = self._object_store.Get(str(version)).Get()

[not at google - send to devlin, 2013/05/15 08:15:00]

shouldn't need to stringify version here

[epeterson, 2013/06/02 00:25:50]

This has been removed completely. AvailabilityFinder's object store is only used
in GetApiAvailability.

+    if names is not None:
+      return api_name in names
+    svn_file_system = self._create_file_system(

[not at google - send to devlin, 2013/05/15 08:15:00]

use the compiled file system for this instead?

[epeterson, 2013/06/02 00:25:50]

Done, although the logic's been moved into a function up there ^^

+        self._chrome_version_utility.GetBranchNumberForVersion(version))
+    names = self._GetAllNames(
+        svn_file_system.ReadSingle('%s/' % svn_constants.API_PATH))
+    self._object_store.Set(str(version), names)
+    return api_name in names
+
+  def _CheckExtensionAPIExistence(self, api_name, version):
+    '''Parses the extension_api.json file (available in earlier versions of
+    chrome) for an API namespace. If this is successfully found, then the API
+    is considered to have been 'stable' for the given version.
+    '''
+    # The extension_api.json files only appear from version 5 to version 17.
+    assert(version <= 17 and version >= 5)
+
+    compiled_fs = self._CreateCompiledFS(
+        self._chrome_version_utility.GetBranchNumberForVersion(version))
+    try:
+      extension_api = self._GetJSONForExtensionAPI(api_name, compiled_fs)
+      api_rows = [row.get('namespace') for row in extension_api
+                  if 'namespace' in row]
+      return True if api_name in api_rows else False
+    except FileNotFoundError as e:
+      # This should only happen on preview.py, since extension_api.json is no
+      # longer present in current versions.
+      return False
+
+  def _FindEarliestAvailability(self, api_name, version):
+    '''Searches in descending order through filesystem caches tied to specific
+    chrome version numbers and looks for the availability of an API, |api_name|,
+    on the stable channel. When a version is found where the API is no longer
+    available on stable, returns the previous version number (the last known
+    version where the API was stable).
+    '''
+    # The _permission_features.json file (with 'channel' keys) is present only
+    # in Chrome 20 and onwards. We'll also check the _manifest_features.json if
+    # an API isn't found in _permission_features, and, failing that, we'll check
+    # for the API's existence in the filesystem.
+    while version >= 20:
+      if ((api_name in self._permission_apis
+          and self._CheckChannelPermissionExistence(api_name,
+                                                    version,
+                                                    'stable'))
+        or (api_name in self._manifest_apis
+            and self._CheckManifestExistence(api_name, version))
+        or (api_name in self._orphan_apis
+            and self._CheckFileSystemExistence(api_name, version))):
+        version -= 1
+      else:
+        return version + 1

[not at google - send to devlin, 2013/05/15 08:15:00]

pls see earlier comments about simplifying the return statements in this
function

+    # These versions are a little troublesome. Version 19 has
+    # _permission_features.json, but it lacks 'channel' information. Version 18
+    # doesn't have either of the json files. For now, we're using a simple check
+    # for filesystem existence here.
+    while version >= 18:
+      if self._CheckFileSystemExistence(api_name, version):
+        version -= 1
+      else:
+        return version + 1
+    # Versions 17 and earlier have an extension_api.json file which contains
+    # namespaces for each API that was available at the time. We can use this
+    # file to check for existence; however, we no longer have
+    # file system access after version 5, so the search stops there.
+    while version >= 5 and self._CheckExtensionAPIExistence(api_name, version):
+      version -= 1
+    return version + 1
+
+  def _IsAvailable(self, api_name, version):

[not at google - send to devlin, 2013/05/15 08:15:00]

yes, except pass this an api and a file system, not an api and a version. that
would require you to write CheckSimplePermissionExistence etc to take a file
system.

[epeterson, 2013/06/02 00:25:50]

The functions for checking existence now take a file system. Since checking
_features files as well as file system existence requires different behavior,
two compiled file systems with unique populate functions are needed. I thought
it looked cleaner to create these compiled fs's here, instead of in the loop in
GetApiAvailability and passing them in, although this can be changed.

+    return ((api_name in self._permission_apis
+        and self._CheckSimplePermissionExistence(api_name, version))
+      or (api_name in self._manifest_apis
+          and self._CheckManifestExistence(api_name, version))
+      or (api_name in self._orphan_apis
+          and self._CheckFileSystemExistence(api_name, version)))
+
+  def GetApiAvailability(self, api_name):
+    '''Determines the availability for an API by testing several scenarios.
+    (i.e. Is the API experimental? Only available on certain development
+    channels? If it's stable, when did it first become stable? etc.)
+    '''
+    availability = self._object_store.Get(api_name).Get()
+    if availability is not None:
+      return self.AvailabilityInfo(availability[0], availability[1])
+
+    # Check for a predetermined availability for this API.
+    availability = self._GetExistingAvailability(api_name, self._json_cache)
+    if availability is not None:
+      self._object_store.Set(api_name, availability)
+      return self.AvailabilityInfo(availability[0], availability[1])

[not at google - send to devlin, 2013/05/15 08:15:00]

this block of code seems to be doing the same as the block before it.

[epeterson, 2013/06/02 00:25:50]

The previous block is checking the object_store, while this block is checking
the api_availabilities.json file. I think this is a little more obvious with the
current code.

+
+    latest_version = self._chrome_version_utility.GetLatestVersionNumber()
+
+    # TODO(epeterson) This shouldn't be done upfront

[not at google - send to devlin, 2013/05/15 08:15:00]

yeah I think this block of code can basically be deleted.

[epeterson, 2013/06/02 00:25:50]

Done.

+    # There are three different scenarios for APIs in Chrome 20 and up. They
+    # can be in _permission_features, _manifest_features, or neither of the two.
+    # APIs are added to a respective list depending on which of these scenarios
+    # they fall into within the trunk filesystem.
+    if self._CheckSimplePermissionExistence(api_name, 'trunk'):
+       # This is a special case: availability can be any of the dev channels.
+       # First, check for this case before checking for stable existence.
+       self._permission_apis.append(api_name)
+    elif self._CheckManifestExistence(api_name, 'trunk'):
+      self._manifest_apis.append(api_name)
+    elif self._CheckFileSystemExistence(api_name, 'trunk'):
+      self._orphan_apis.append(api_name)
+    else:
+      raise FileNotFoundError(
+          'The API schema for %s could not be located.' % api_name)
+
+    # Channel: availability[0]; Version: availability[1]
+    availability = ['trunk', latest_version]
+    for channel in BranchUtility.GetAllChannelNames():
+      channel_info = self._branch_utility.GetChannelInfo(channel)
+      # TODO(epeterson) unicode to ints in branch_utility
+      if self._IsAvailable(api_name, availability[1]):
+        availability[0] = channel
+        if channel_info.version == 'trunk':
+          availability[1] = 'trunk'
+        else:
+          availability[1] = int(channel_info.version)
+        break

[not at google - send to devlin, 2013/05/15 08:15:00]

this all looks very complicated?

I think it can be:

availability = None
for channel_info in self._branch_utility.GetAllChannelInfo():
  if self._IsAvailableInFileSystem(
      api_name, self._create_file_system(channel_info.branch)):
    availability = AvailabilityFinder.AvailabilityInfo(channel_info.channel,
                                                       channel_info.version)
    break

# The API should at least be available on trunk. It's a bug otherwise.
assert availability, 'Did not find any availability for %s' % channel

# If the API is in stable, find the chrome version which it became stable in.
if availability.channel == 'stable':
  availability.version = self._FindEarliestAvailability(api_name,
                                                        availability.version)

[epeterson, 2013/06/02 00:25:50]

Done.

+    if availability[0] == 'stable':
+      availability[1] = self._FindEarliestAvailability(
+          api_name,
+          int(availability[1]))
+
+    self._object_store.Set(api_name, availability)
+    return self.AvailabilityInfo(availability[0], availability[1])