Dynamically generate a heading for Extension Docs API pages

chrome/common/extensions/docs/server2/availability_data_source.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import os
+
+from branch_utility import BranchUtility
+from chrome_version_utility import ChromeVersionUtility
+from file_system import FileNotFoundError
+import svn_constants
+from third_party.json_schema_compiler import json_parse, model
+
+EXTENSION_API = 'extension_api.json'
+MANIFEST_FEATURES = '_manifest_features.json'
+PERMISSION_FEATURES = '_permission_features.json'
+
+class AvailabilityDataSource(object):

[not at google - send to devlin, 2013/05/13 21:26:41]

Late stage for this, but this isn't actually a *DataSource* which by convention
implies an object which is exposed to templates. That has been confusing me
somewhat. Let's call it AvailabilityFinder.

[epeterson, 2013/05/15 07:38:34]

Done.

+  '''Uses API data sources generated by a ChromeVersionDataSource in order to
+  search the filesystem for the earliest existence of a specified API throughout
+  the different versions of Chrome; this constitutes an API's availability.
+  '''
+  class Factory(object):
+    def __init__(self,
+                 chrome_version_utility,
+                 object_store_creator,
+                 create_file_system):

[not at google - send to devlin, 2013/05/13 21:26:41]

Can we try making this a method of ChromeVersionUtility? To create a host file
system for a given version?

+      self._chrome_version_utility = chrome_version_utility
+      self._object_store_creator = object_store_creator
+      self._create_file_system = create_file_system
+
+    def Create(self):
+      return AvailabilityDataSource(self._chrome_version_utility,
+                                    self._object_store_creator,
+                                    self._create_file_system)
+
+  def __init__(self,
+               chrome_version_utility,
+               object_store_creator,
+               create_file_system):
+    self._chrome_version_utility = chrome_version_utility
+    self._object_store_creator = object_store_creator
+    self._create_file_system = create_file_system
+
+    svn_file_system = create_file_system('trunk')
+    def read_json(path):
+      return json_parse.Parse(svn_file_system.ReadSingle(path))
+    self._existing_availabilities = read_json(
+        '%s/api_availabilities.json' % svn_constants.JSON_PATH)
+    self._permission_features = read_json(
+        '%s/%s' % (svn_constants.API_PATH, PERMISSION_FEATURES))
+    self._manifest_features = read_json(
+        '%s/%s' % (svn_constants.API_PATH, MANIFEST_FEATURES))

[not at google - send to devlin, 2013/05/13 15:43:32]

here is one problem. The file systems are being read, but they might be offline.
All data needs to originate from the object stores (compiled file systems solve
this problem automatically by the way).

[not at google - send to devlin, 2013/05/13 21:26:41]

Continuing from this comment, and reading below: I think you can get away
without injecting any file systems into here: (a) move the getting of a file
system into ChromeVersionUtility, and (b) use a CompiledFileSystem that calls
json_parse.Parse for all of these. You'll need to inject the paths though.

def __init__(self,
            chrome_version_util,
            object_store_creator,
            compiled_fs_factory,
            api_availabilities_path,
            permission_features_path,
            manifest_features_path,
            extension_api_path):
  self._json_cache = compiled_fs_factory.Create(json_parse.Parse,
AvailabilityDataSource)
  self._ ....

...

then instead of

self._permisison_features.get(api_name)

do

self._GetPermissionFeature(api_name)

with

def _GetPermissionFeature(self, api_name):
  return self._json_cache.GetFromFile(
      self._permission_features_path).get(api_name)

+
+    def create_object_store(category):
+      return (object_store_creator.Create(AvailabilityDataSource,
+                                          category=category))
+    self._permission_object_store = create_object_store('permission')
+    self._manifest_object_store = create_object_store('manifest')
+    self._names_object_store = create_object_store('names')
+    self._object_store = create_object_store(None)
+
+    self._permission_apis = []
+    self._manifest_apis = []
+    self._orphan_apis = []

[not at google - send to devlin, 2013/05/13 21:26:41]

Looking at the way this is used, it's a bit strange to be holding onto this data
like that, but I hope they'll go away when you address the comments in this
file.

+
+  def _ReadJson(self, svn_file_system, path):
+    return json_parse.Parse(svn_file_system.ReadSingle(path))
+
+  def _GetChannelFromPermissionFeatures(self, api_name, permission_features):

[not at google - send to devlin, 2013/05/13 21:26:41]

this function should be able to use self._permission_features, not take it as a
parameter?

[epeterson, 2013/05/15 07:38:34]

Done.

+    '''Handles finding API channel information from _permission_features.json.
+    Sometimes this info will be in a dict, but other times it will be
+    in a dict contained in a list.
+    '''
+    channel = None
+    api_info = permission_features.get(api_name)
+    if api_info is not None:
+      if isinstance(api_info, collections.Mapping):
+        channel = api_info.get('channel')
+      else:
+        channel = BranchUtility.NewestChannel([entry.get('channel')
+                                               for entry in api_info])

[not at google - send to devlin, 2013/05/13 21:26:41]

I don't think that the list comprehension is necessary, just passing the
generator in should work ok.

channel = BranchUtility.NewestChannel(entry.get('channel') for entry in
api_info)

[epeterson, 2013/05/15 07:38:34]

Done.

+    return channel
+
+  def _CheckStablePermissionExistence(self, api_name, version):
+    '''Checks _permission_features.json for the given |version| of Chrome,
+    and returns a boolean representing whether or not the API was available on
+    the stable channel in |version|.
+    '''
+    existence = self._permission_object_store.Get(
+        '%s%s' % (api_name, version)).Get()
+    if existence is not None:
+      return existence
+    svn_file_system = self._create_file_system(
+        self._chrome_version_utility.GetBranchNumberForVersion(version))
+    permission_features = self._ReadJson(
+        svn_file_system,
+        '%s/%s' % (svn_constants.API_PATH, PERMISSION_FEATURES))
+    existence = 'stable' == self._GetChannelFromPermissionFeatures(
+        api_name,
+        permission_features)
+    self._object_store.Set('%s%s' % (api_name, version), existence)
+    return existence
+
+  def _CheckManifestExistence(self, api_name, version):
+    '''As a fallback measure for an API not being found in _permission_features,
+    check _manifest_features.json for the given |version| of Chrome,
+    and return a boolean representing whether or not the API was represented
+    within the file.
+    '''
+    #_manifest_features.json uses unix_hacker_style API names.
+    api_name = model.UnixName(api_name)
+    existence = self._manifest_object_store.Get(
+        '%s%s' % (api_name, version)).Get()
+    if existence is not None:
+      return existence
+    svn_file_system = self._create_file_system(
+        self._chrome_version_utility.GetBranchNumberForVersion(version))
+    manifest_features = self._ReadJson(
+        svn_file_system,
+        '%s/%s' % (svn_constants.API_PATH, MANIFEST_FEATURES))
+    if manifest_features.get(api_name) is not None:
+      existence = True
+    else:
+      existence = False

[not at google - send to devlin, 2013/05/13 21:26:41]

existence = api_name in manifest_features

[epeterson, 2013/05/15 07:38:34]

Done.

+    self._object_store.Set('%s%s' % (api_name, version), existence)
+    return existence
+
+  def _GetAllNames(self, files_dict):
+    '''Returns extension-less names for a list of files.
+    '''
+    files = files_dict[svn_constants.API_PATH + '/']
+    return [os.path.splitext(f)[0] for f in files]
+
+  def _CheckFileSystemExistence(self, api_name, version):
+    '''Uses a list of API names for a given branch to check simply for the
+    existence of a given API in that branch.
+    '''
+    names = self._names_object_store.Get(version).Get()
+    if names is not None:
+      return api_name in names
+    svn_file_system = self._create_file_system(
+        self._chrome_version_utility.GetBranchNumberForVersion(version))
+    names = svn_file_system.Read([svn_constants.API_PATH + '/']).Get()

[not at google - send to devlin, 2013/05/13 21:26:41]

ReadSingle

search in this file for ".Read(" and see if they can be ReadSIngle.

[epeterson, 2013/05/15 07:38:34]

Done.

+    names = self._GetAllNames(names)
+    self._object_store.Set(version, names)
+    return api_name in names
+
+  def _CheckExtensionAPIExistence(self, api_name, version):
+    '''Parses the extension_api.json file (available in earlier versions of
+    chrome) for an API namespace. If this is successfully found, then the API
+    is considered to have been 'stable' for the given version.
+    '''
+    # The extension_api.json files only appear from version 5 to version 17.
+    if int(version) > 17 or int(version) < 5:

[not at google - send to devlin, 2013/05/13 21:26:41]

make these assert, this method should only be called under the right
circumstances.

[epeterson, 2013/05/15 07:38:34]

Done.

+      return False
+
+    # Re-use this object-store - the versions won't conflict with each other.
+    existence = self._names_object_store.Get('%s%s' % (api_name, version)).Get()

[not at google - send to devlin, 2013/05/13 21:26:41]

I don't think caching needs to be done here, you're already caching the result
of GetAvailability. It just complicates things?

[epeterson, 2013/05/15 07:38:34]

Done.

+    if existence is not None:
+      return existence
+    svn_file_system = self._create_file_system(
+        self._chrome_version_utility.GetBranchNumberForVersion(version))
+    try:
+      extension_api = self._ReadJson(
+          svn_file_system,
+          '%s/%s' % (svn_constants.API_PATH, EXTENSION_API))
+      api_rows = [row.get('namespace') for row in extension_api
+                  if 'namespace' in row]
+      return True if api_name in api_rows else False
+    except FileNotFoundError as e:
+      # This should only happen on preview.py, since extension_api.json is no
+      # longer present in current versions.
+      return False
+
+  def _FindEarliestStableAvailability(self, api_name, version):
+    '''Searches in descending order through filesystem caches tied to specific
+    chrome version numbers and looks for the availability of an API, |api_name|,
+    on the stable channel. When a version is found where the API is no longer
+    available on stable, returns the previous version number (the last known
+    version where the API was stable).
+    '''
+    # The _permission_features.json file (with 'channel' keys) is present only
+    # in Chrome 20 and onwards. We'll also check the _manifest_features.json if
+    # an API isn't found in _permission_features, and, failing that, we'll check
+    # for the API's existence in the filesystem.
+    while version >= 20:
+      if ((api_name in self._permission_apis
+          and self._CheckStablePermissionExistence(api_name, str(version)))
+        or (api_name in self._manifest_apis
+            and self._CheckManifestExistence(api_name, str(version)))
+        or (api_name in self._orphan_apis
+            and self._CheckFileSystemExistence(api_name, str(version)))):
+        version -= 1
+      else:
+        break

[not at google - send to devlin, 2013/05/13 21:26:41]

THis method is super cool, but a bit odd to read all of the extra while
conditions. Just make it so that every time you find the version return it, then
invert the conditions to do so, then not have all of the < X checks.

[epeterson, 2013/05/15 07:38:34]

Done.

+    # These versions are a little troublesome. Version 19 has
+    # _permission_features.json, but it lacks 'channel' information. Version 18
+    # doesn't have either of the json files. For now, we're using a simple check
+    # for filesystem existence here.
+    while version < 20 and version >= 18:
+      if self._CheckFileSystemExistence(api_name, str(version)):
+        version -= 1
+      else:
+        break
+    # Versions 17 and earlier have an extension_api.json file which contains
+    # namespaces for each API that was available at the time. We can use this
+    # file to check for existence, however, we no longer have
+    # file system access after version 5, so stop there.
+    while version < 18 and version >= 5:
+      if self._CheckExtensionAPIExistence(api_name, str(version)):

[not at google - send to devlin, 2013/05/13 21:26:41]

why str(version)? Can we use ints everywhere?

[epeterson, 2013/05/15 07:38:34]

Done.

+        version -= 1
+      else:
+        break
+    return str(version + 1)
+
+  def GetAvailability(self, api_name):

[not at google - send to devlin, 2013/05/13 21:26:41]

I will want to add event/function/property availability as well, so perhaps call
this GetApiAvailability.

[epeterson, 2013/05/15 07:38:34]

Done.

+    '''Determines the availability for an API by testing several scenarios.
+    (i.e. Is the API experimental? Only available on certain development
+    channels? If it's stable, when did it first become stable? etc.)
+    '''
+    availability = self._object_store.Get(api_name).Get()
+    if availability is not None:
+      return availability
+
+    # Check for a predetermined availability for this API.
+    availability = self._existing_availabilities.get(api_name)

[not at google - send to devlin, 2013/05/13 21:26:41]

e.g. in lazy creation world this would be
self._GetExistingAvailability(api_name)

and _GetExistingAvailability would know how to call ReadFromFile etc. Then
CompiledFileSystem implements the caching that you would otherwise need to
implement yourself.

[epeterson, 2013/05/15 07:38:34]

Done.

+    if availability is not None:
+      self._object_store.Set(api_name, availability)
+      return availability
+
+    latest_version = str(self._chrome_version_utility.GetLatestVersionNumber())
+    channel = self._GetChannelFromPermissionFeatures(api_name,
+                                                     self._permission_features)

[not at google - send to devlin, 2013/05/13 21:26:41]

This won't actually work properly, though: say if an API has just been promoted
from dev to beta, like, 10 minutes ago. This method will return 'beta', however,
in the actual beta channel it's still dev restricted, so won't work.

I think you need to step through all channels, starting from stable, until you
reach a point where it's available:

branch_util = self._branch_util
version_util = self._chrome_version_util

availability = ('trunk', version_util.GetLatestVersionNumber())
for channel in BranchUtility.GetChannels():
  if self._IsAvailable(api_name, self._GetFileSystemForChannel(channel)):
    availability = (channel, branch_util.GetChannelInfo(channel).version)
    break
if availability[0] == 'stable':
  availability[1] = self._FindEarliestAvailability(api, availability[1])
self._object_store.Set(api_name, availability)
return availability

then self._IsAvailable does what you're doing here pretty much; check
permission, manifest, then file system. I suppose it would end up looking very
similar to _FindEarliestAvailability - that does all the right things, except it
only works for stable - it needs to work for all the channels.

In the above sample I actually made availability be a (channel, version) tuple,
where channel is obvious, and version is the Chrome version that it first
appeared in that channel (only really useful for stable - but let callers decide
that). I think this would be clearer. It would also avoid the conflation of
channel and version that this method currently suffers from.

[epeterson, 2013/05/15 07:38:34]

So this is a bit of a mess now. Couldn't quite get it figured out tonight.

+
+    # There are three different scenarios for APIs in Chrome 20 and up. They
+    # can be in _permission_features, _manifest_features, or neither of the two.
+    # APIs are added to a respective list depending on which of these scenarios
+    # they fall into within the trunk filesystem.
+    if channel is not None:
+       # This is a special case: availability can be any of the dev channels.
+       # First, check for this case before checking for stable existence.
+       availability = channel
+       self._permission_apis.append(api_name)
+    elif self._CheckManifestExistence(api_name, latest_version):
+      availability = 'stable'
+      self._manifest_apis.append(api_name)
+    elif self._CheckFileSystemExistence(api_name, latest_version):
+      availability = 'stable'
+      self._orphan_apis.append(api_name)
+    else:
+      raise FileNotFoundError(
+          'The API schema for %s could not be located.' % api_name)
+      availability = None
+
+    if availability == 'stable':
+      availability = self._FindEarliestStableAvailability(
+          api_name,
+          int(latest_version) - 1)
+    self._object_store.Set(api_name, availability)
+    return availability