Dynamically generate a heading for Extension Docs API pages

chrome/common/extensions/docs/server2/availability_finder.py

Index: chrome/common/extensions/docs/server2/availability_finder.py
diff --git a/chrome/common/extensions/docs/server2/availability_finder.py b/chrome/common/extensions/docs/server2/availability_finder.py
new file mode 100644
index 0000000000000000000000000000000000000000..5acd7f742dee3cc5f39cfcf7db4f5b277ebded46
--- /dev/null
+++ b/chrome/common/extensions/docs/server2/availability_finder.py
@@ -0,0 +1,235 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import logging
+import os
+
+from branch_utility import BranchUtility
+from compiled_file_system import CompiledFileSystem
+from file_system import FileNotFoundError
+import svn_constants
+from third_party.json_schema_compiler import json_parse, model
+
+_API_AVAILABILITIES = svn_constants.JSON_PATH + '/api_availabilities.json'
+_EXTENSION_API = svn_constants.API_PATH + '/extension_api.json'
+_API_FEATURES = svn_constants.API_PATH + '/_api_features.json'
+_MANIFEST_FEATURES = svn_constants.API_PATH + '/_manifest_features.json'
+_PERMISSION_FEATURES = svn_constants.API_PATH + '/_permission_features.json'
+
+def _GetChannelFromFeature(feature):
+  '''Handles finding API channel information from _features.json files.

[not at google - send to devlin, 2013/06/05 00:24:09]

"Finds API channel information..."

[epeterson, 2013/06/17 20:05:49]

Done.

+  Sometimes this info will be in a dict, but other times it will be
+  in a dict contained in a list for whitelisting purposes.
+  '''
+  channel = None
+  if feature is not None:
+    if isinstance(feature, collections.Mapping):
+      channel = feature.get('channel')
+    else:
+      channel = BranchUtility.NewestChannel(entry.get('channel')
+                                            for entry in feature)
+  return channel

[not at google - send to devlin, 2013/06/05 00:24:09]

more concise:

if feature is None:
  return None
if isinstance(feature, collections.Mapping):
  return feature.get('channel')
# Features can contain a list of entries. Take the newest branch.
return BranchUtility.NewestChannel(entry.get('channel')
                                   for entry in feature)

[epeterson, 2013/06/17 20:05:49]

Done.

+

[epeterson, 2013/06/02 00:25:50]

A lot of the logic for checking individual files/file existence was moved out of
the AvailabilityFinder class. I also tried to make this generic enough to allow
for the possibility of additional, similar _features files being added in the
future (like _api_features.json)

+def _GetFeature(api_name, file_system, path, unix_name=False):

[not at google - send to devlin, 2013/06/05 00:24:09]

this is only used in ExistsInFeatures, move it in there.

[epeterson, 2013/06/17 20:05:49]

Done. It might be useful to access the rest of the feature information besides
just 'channel' in the future, though.

+  '''Gets the data for a given API schema from api/_api_features.json
+  within the given file system.
+  '''
+  if unix_name:
+    api_name = model.UnixName(api_name)

[not at google - send to devlin, 2013/06/05 00:24:09]

what is this unix_name thing?

[epeterson, 2013/06/17 20:05:49]

_manifest_features.json uses unix_style API names, so the name needs to be
converted.
I'll document this.

+  return file_system.GetFromFile(path).get(api_name)
+
+def _ExistsInFeatures(api_name, file_system, path, unix_name=False):

[not at google - send to devlin, 2013/06/05 00:24:09]

as per comment way below, call all of these methods like _GetChannelFromFeatures
and return the channel from all of them (or None).

[epeterson, 2013/06/17 20:05:49]

Done.

+  '''Returns the name of the development channel that the API is available on
+  if this information can be located in the _features.json file specified by
+  |path| within the given |file_system|. Returns None, otherwise.
+  '''
+  return _GetChannelFromFeature(
+      _GetFeature(api_name, file_system, path, unix_name))
+
+def _ExistsInApiFeatures(api_name, file_system):
+  try:
+    return _ExistsInFeatures(api_name, file_system, _API_FEATURES)
+  except FileNotFoundError as e:
+    # TODO(epeterson) Remove except block once _api_features is in all channels.
+    logging.warning('_api_features.json not found')

[not at google - send to devlin, 2013/06/05 00:24:09]

don't warn it'll pollute the logs - but good point.

[epeterson, 2013/06/17 20:05:49]

Done.

+    return False
+
+def _ExistsInPermissionFeatures(api_name, file_system):
+  return _ExistsInFeatures(api_name, file_system, _PERMISSION_FEATURES)
+
+def _ExistsInManifestFeatures(api_name, file_system):

[not at google - send to devlin, 2013/06/05 00:24:09]

_GetChannelFromManifestFeatures

[epeterson, 2013/06/17 20:05:49]

Done.

+  return _ExistsInFeatures(api_name, file_system, _MANIFEST_FEATURES)
+
+def _ExistsInFileSystem(api_name, file_system):
+  '''Checks for existence of |api_name| within the list of files in the api/
+  directory found using the given file system.
+  '''
+  api_name = model.UnixName(api_name)
+  return api_name in file_system.GetFromFileListing(svn_constants.API_PATH)
+
+def _ExistsInExtensionApi(api_name, file_system):
+  '''Parses the api/extension_api.json file (available in Chrome versions
+  before 18) for an API namespace. If this is successfully found, then the API
+  is considered to have been 'stable' for the given version.
+  '''
+  try:
+    extension_api_json = file_system.GetFromFile(_EXTENSION_API)
+    api_rows = [row.get('namespace') for row in extension_api_json
+                if 'namespace' in row]
+    return True if api_name in api_rows else False
+  except FileNotFoundError as e:
+    # This should only happen on preview.py since extension_api.json is no
+    # longer present in trunk.
+    return False
+
+class AvailabilityFinder(object):
+  '''Uses API data sources generated by a ChromeVersionDataSource in order to
+  search the filesystem for the earliest existence of a specified API throughout
+  the different versions of Chrome; this constitutes an API's availability.
+  '''
+  class Factory(object):
+    def __init__(self,
+                 object_store_creator,
+                 compiled_host_fs_factory,
+                 branch_utility,
+                 create_file_system):

[not at google - send to devlin, 2013/06/05 00:24:09]

please document create_file_system. create_file_system_at_version might be a
better name if that's what it does.

[epeterson, 2013/06/17 20:05:49]

Done.

+      self._object_store_creator = object_store_creator
+      self._compiled_host_fs_factory = compiled_host_fs_factory
+      self._branch_utility = branch_utility
+      self._create_file_system = create_file_system
+
+    def Create(self):
+      return AvailabilityFinder(self._object_store_creator,
+                                self._compiled_host_fs_factory,
+                                self._branch_utility,
+                                self._create_file_system)
+
+  def __init__(self,
+               object_store_creator,
+               compiled_host_fs_factory,
+               branch_utility,
+               create_file_system):
+    self._object_store_creator = object_store_creator
+    self._json_cache = compiled_host_fs_factory.Create(
+        lambda _, json: json_parse.Parse(json),
+        AvailabilityFinder,
+        'json-cache')
+    self._branch_utility = branch_utility
+    self._create_file_system = create_file_system
+    self._object_store = object_store_creator.Create(AvailabilityFinder)
+
+  def _CreateFeaturesAndNamesFileSystems(self, version):

[epeterson, 2013/06/02 00:25:50]

Checking _features.json files requires a different populate function for a
compiled fs than checking a list of file names (checking for file existence).
These file systems are created and returned together, though, since they're
always used in the same places. 

+    '''The 'features' compiled file system's populate function parses and
+    returns the contents of a _features.json file. The 'names' compiled file
+    system's populate function creates a list of file names with .json or .idl
+    extensions.
+    '''
+    fs_factory = CompiledFileSystem.Factory(

[not at google - send to devlin, 2013/06/05 00:24:09]

recreating them each time like this is inefficent because we use in-memory
caches which will need to be repopulated each time.

but adding @memoize to this function will do it (from
third_party.json_schema_compiler)

[epeterson, 2013/06/17 20:05:49]

Done.

+        self._create_file_system(version),
+        self._object_store_creator)
+    features_fs = fs_factory.Create(lambda _, json: json_parse.Parse(json),
+                                    AvailabilityFinder,
+                                    '%sFeatures' % version)

[not at google - send to devlin, 2013/06/05 00:24:09]

category=

also you don't need the version since the compiled file system is constructed
with it. just 'features'.

[epeterson, 2013/06/17 20:05:49]

Ok, this clears things up. thank you.

+    names_fs = fs_factory.Create(self._GetExtNames,
+                                 AvailabilityFinder,
+                                 '%sNames' % version)

[not at google - send to devlin, 2013/06/05 00:24:09]

likewise category='names'

[epeterson, 2013/06/17 20:05:49]

Done.

+    return (features_fs, names_fs)
+
+  def _GetExtNames(self, base_path, apis):
+    return [os.path.splitext(api)[0] for api in apis
+            if os.path.splitext(api)[1][1:] in ['json', 'idl']]
+
+  def _FindEarliestAvailability(self, api_name, version):
+    '''Searches in descending order through filesystem caches tied to specific
+    chrome version numbers and looks for the availability of an API, |api_name|,
+    on the stable channel. When a version is found where the API is no longer
+    available on stable, returns the previous version number (the last known
+    version where the API was stable).
+    '''

[epeterson, 2013/06/02 00:25:50]

I think this code looks a lot more readable, and simplifies the return logic.
It's recursive, though, and that might be taboo..

[not at google - send to devlin, 2013/06/05 00:24:09]

Yes much better, though recursiveness should be avoided if it's just as
reasonable to do it iteratively. Which, here it is. I'm pretty sure the
algorithm I wrote for you would have more or less worked fine?

[epeterson, 2013/06/17 20:05:49]

Here's an iterative version.

+    if version >= 5:
+      features_fs, names_fs = self._CreateFeaturesAndNamesFileSystems(version)

[not at google - send to devlin, 2013/06/05 00:24:09]

just have an "if version < 5: return version" at the top.

[epeterson, 2013/06/17 20:05:49]

Done.

+
+    available = False
+    # The _api_features.json file first appears in version 28 and should be the
+    # most reliable for finding API availabilities, so it gets checked first.
+    # The _permission_features.json and _manifest_features.json files are
+    # present in Chrome 20 and onwards. Fall back to a check for file system
+    # existence if the API is not stable in any of the _features.json files.

[not at google - send to devlin, 2013/06/05 00:24:09]

could you put all of these comments inside the blocks that guard them? it would
be easier to read.

[epeterson, 2013/06/17 20:05:49]

Done.

+    if version >= 28:
+      available = _ExistsInApiFeatures(api_name, features_fs) == 'stable'
+    # Check other _features.json files/file existence if the API wasn't found in
+    # _api_features.json, or if _api_features.json wasn't present.
+    if version >= 20:
+      available = available or (
+          _ExistsInPermissionFeatures(api_name, features_fs) == 'stable'
+          or _ExistsInManifestFeatures(api_name, features_fs) == 'stable'
+          or _ExistsInFileSystem(api_name, names_fs))
+    # These versions are a little troublesome. Version 19 has
+    # _permission_features.json, but it lacks 'channel' information. Version 18
+    # lacks all of the _features.json files. For now, we're using a simple check
+    # for filesystem existence here.
+    elif version >= 18:
+      available = _ExistsInFileSystem(api_name, names_fs)
+    # Versions 17 and down to 5 have an extension_api.json file which contains
+    # namespaces for each API that was available at the time. We can use this
+    # file to check for API existence.
+    elif version >= 5:
+      available = _ExistsInExtensionApi(api_name, features_fs)
+
+    if available:
+      return self._FindEarliestAvailability(api_name, version - 1)
+    return version + 1
+
+  def _IsAvailableInFileSystem(self, api_name, version):

[not at google - send to devlin, 2013/06/05 00:24:09]

as noted below, make this _GetAvailableChannelForVersion

[epeterson, 2013/06/17 20:05:49]

Done.

+    features_fs, names_fs = self._CreateFeaturesAndNamesFileSystems(version)
+    return (_ExistsInApiFeatures(api_name, features_fs)
+            or _ExistsInPermissionFeatures(api_name, features_fs)
+            or _ExistsInManifestFeatures(api_name, features_fs)
+            or _ExistsInFileSystem(api_name, names_fs))
+
+  def GetApiAvailability(self, api_name):
+    '''Determines the availability for an API by testing several scenarios.
+    (i.e. Is the API experimental? Only available on certain development
+    channels? If it's stable, when did it first become stable? etc.)
+    '''
+    availability = self._object_store.Get(api_name).Get()
+    if availability is not None:
+      return availability
+
+    # Check for a predetermined availability for this API.
+    api_info = self._json_cache.GetFromFile(_API_AVAILABILITIES).get(api_name)
+    if api_info is not None:
+      channel = api_info.get('channel')
+      version = api_info.get('version') if channel == 'stable' else None
+      availability = AvailabilityInfo(channel, version)
+      self._object_store.Set(api_name, availability)
+      return availability

[not at google - send to devlin, 2013/06/05 00:24:09]

more concise:

availability = AvailabilityInfo(api_info['channel'], api_info['version'])
self._object_store.Set(api_name, availability)
return availability

in fact there isn't much point caching it since json is cached anyway and this
stuff is basically free to look up. so this can all be just

if api_info is not None:
  return AvailabilityInfo(...)

[epeterson, 2013/06/17 20:05:49]

Done.

+
+    # Check for the API in the development channels.
+    availability = None
+    for channel_info in self._branch_utility.GetAllChannelInfo():
+      if self._IsAvailableInFileSystem(api_name,
+                                       channel_info.version):
+        availability = AvailabilityInfo(channel_info.channel,
+                                        channel_info.version)
+        break

[not at google - send to devlin, 2013/06/05 00:24:09]

i think this has a bug in scenarios such as: an API is present in the stable
build but only available to dev channel. This algorithm will report stable, but
it's really only dev.

Try:

for channel_info in self._branch_utility.GetAllChannelInfo():
  available_channel = self._GetAvailableChannelForVersion(
      channel_info.version)
  if (available_channel is not None and
      BranchUtility.NewestVersion((available_channel,
                                   channel_info.channel))
          == available_channel):
    availability = AvailabilityInfo(channel_info.channel,
                                    channel_info.version)
    break

and please test this!

[epeterson, 2013/06/17 20:05:49]

Going to continue testing as I prepare to submit a patch devoted to
AvailabilityFixer and its test.

In the meantime, it looks like the if-test needs to be:
  if (available_channel is not None and
      BranchUtility.NewestVersion((available_channel,
                                   channel_info.channel))
          == channel_info.channel):

For example:
  - notifications is listed as 'dev' on beta and 'beta' on dev.
  - beta is checked first, and the available channel, 'dev', will be newer than
'beta'; this fails the NewestVersion(...) == channel_info.channel
  - dev is checked next, with the available_channel as 'beta' and
channel_info.channel as 'dev'. NewestVersion(...) will return 'dev', which will
pass the if-check.
  - notifications is available on the dev channel

+
+    # The API should at least be available on trunk. It's a bug otherwise.
+    assert availability, 'No availability found for %s' % api_name
+
+    # If the API is in stable, find the chrome version in which it became
+    # stable.
+    if availability.channel == 'stable':
+      availability.version = self._FindEarliestAvailability(
+          api_name,
+          availability.version)
+
+    self._object_store.Set(api_name, availability)
+    return availability
+
+class AvailabilityInfo(object):

[not at google - send to devlin, 2013/06/05 00:24:09]

declare at top

[epeterson, 2013/06/17 20:05:49]

Done.

+  def __init__(self, channel, version):
+    self.channel = channel
+    self.version = version