Dynamically generate a heading for Extension Docs API pages

chrome/common/extensions/docs/server2/availability_data_source.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import logging
+import os
+
+from appengine_url_fetcher import AppEngineUrlFetcher
+from caching_file_system import CachingFileSystem
+from compiled_file_system import CompiledFileSystem
+from subversion_file_system import SubversionFileSystem
+import svn_constants
+from third_party.json_schema_compiler import json_comment_eater
+from third_party.json_schema_compiler import model
+import url_constants
+
+# These labels are used for interacting with AvailabilityDataSource's object
+# store. The format is <label><string> (e.g. svn21, or permissiontabs26)
+SVN = 'svn'
+PERMISSION = 'permission'
+MANIFEST = 'manifest'
+FILESYS = 'filesys'

[not at google - send to devlin, 2013/04/30 18:34:19]

use these in category= when creating the object store rather than namespacing
everything.

[epeterson, 2013/05/13 02:38:10]

Done.

+
+class AvailabilityDataSource(object):
+  '''Uses API data sources generated by a ChromeVersionDataSource in order to
+  search the filesystem for the earliest existence of a specified API throughout
+  the different versions of Chrome; this constitutes an API's availability.
+  '''
+  class Factory(object):
+    def __init__(self,
+                 chrome_version_utility,
+                 object_store_creator_factory,
+                 svn_file_system):
+      self._chrome_version_utility = chrome_version_utility
+      self._object_store_creator_factory = object_store_creator_factory
+      json_string = svn_file_system.ReadSingle('%s/api_availabilities.json'
+                                               % svn_constants.JSON_PATH)
+      self._existing_availabilities = json.loads(json_string)
+      json_string = svn_file_system.ReadSingle('%s/_permission_features.json'
+                                               % svn_constants.API_PATH)
+      self._permission_features = json.loads(json_comment_eater.Nom(
+          json_string))
+      json_string = svn_file_system.ReadSingle('%s/_manifest_features.json'
+                                               % svn_constants.API_PATH)

[not at google - send to devlin, 2013/04/30 18:34:19]

use third_party.json_schema_compiler.json_parse as json_parse here, to get the
ordered_dict behaviour.

nicer way to factor:

def read_json(path):
  return json_parse.Parse(svn_file_system.ReadSingle(path))
self._existing_availability = read_json(...)
self._permission_feature = read_json(...)
self._manifest_features = read_json(...)

[epeterson, 2013/05/13 02:38:10]

Done.

+      self._manifest_features = json.loads(json_comment_eater.Nom(json_string))
+
+    def Create(self):
+      return AvailabilityDataSource(self._chrome_version_utility,
+                                    self._object_store_creator_factory,
+                                    self._existing_availabilities,
+                                    self._permission_features,
+                                    self._manifest_features)
+
+  def __init__(self,
+               chrome_version_utility,
+               object_store_creator_factory,
+               existing_availabilities,
+               permission_features,
+               manifest_features):
+    self._chrome_version_utility = chrome_version_utility
+    self._object_store_creator_factory = object_store_creator_factory
+    self._object_store = self._object_store_creator_factory.Create(
+        AvailabilityDataSource).Create()
+    self._existing_availabilities = existing_availabilities
+    self._permission_features = permission_features
+    self._manifest_features = manifest_features
+    self._permission_apis = []
+    self._manifest_apis = []
+    self._orphan_apis = []
+
+  def _CreateSvnFileSystemForBranch(self, branch):

[not at google - send to devlin, 2013/04/30 18:34:19]

maybe this should go as a little helper on SubversionFileSystem.

class SubversionFileSystem(object):
  @staticmethod
  CreateForBranch(branch):
    ... this

use from ServerInstance.

[epeterson, 2013/05/13 02:38:10]

Well, I had done this, and modified the old CreateOnline/GetOrCreateOffline
methods in ServerInstance. But these methods were separated from ServerInstance
with a previous patch, so I tried to work with that. See the updated code in
cron_servlet, instance_servlet.

+    '''Creates a file system tied to a specific |branch| of the svn repository.
+    '''
+    if branch == 'trunk':
+      svn_url = '/'.join((url_constants.SVN_TRUNK_URL,
+                          'src',
+                          svn_constants.EXTENSIONS_PATH))
+    else:
+      svn_url = '/'.join((url_constants.SVN_BRANCH_URL,
+                          branch,
+                          'src',
+                          svn_constants.EXTENSIONS_PATH))
+    viewvc_url = svn_url.replace(url_constants.SVN_URL,
+                                 url_constants.VIEWVC_URL)
+
+    svn_file_system = CachingFileSystem(
+        SubversionFileSystem(AppEngineUrlFetcher(svn_url),
+                             AppEngineUrlFetcher(viewvc_url)),
+        self._object_store_creator_factory)
+    return svn_file_system
+
+  def _GetChannelFromPermissionFeatures(self, api_name, permission_features):
+    '''Sometimes the API information will be in a dict, other times it will be
+    in a dict contained in a list.
+    '''
+    api_info = permission_features.get(api_name, None)
+    if api_info is not None:
+      try:
+        channel = api_info.get('channel')
+      except AttributeError:
+        channel = api_info[0].get('channel')

[not at google - send to devlin, 2013/04/30 18:34:19]

don't use exceptions for control flow. Also, if it's a list we need to actually
find the min availability - the only reason these are in lists usually is to
whitelist the API in stable for some extension. This shouldn't show up as stable
here, it should show up as dev. Maybe BranchUtility needs to expose a util to
order these, like BranchUtility.Newest(list of branch names)

[epeterson, 2013/05/13 02:38:10]

Done.

+      finally:
+        return channel
+    return None
+
+  def _CheckStablePermissionExistence(self, api_name, version):
+    '''Checks _permission_features.json for the given |version| of Chrome,
+    and returns a boolean representing whether or not the API was available on
+    the stable channel in |version|.
+    '''
+    existence = self._object_store.Get('%s%s%s'
+                                       % (PERMISSION, api_name, version)).Get()
+    if existence is not None:
+      return existence
+
+    svn_file_system = self._object_store.Get('%s%s' % (SVN, version)).Get()
+    if svn_file_system is None:
+      svn_file_system = self._CreateSvnFileSystemForBranch(
+          self._chrome_version_utility.GetBranchNumberForVersion(version))

[not at google - send to devlin, 2013/04/30 18:34:19]

I don't think you can put svn file systems in object stores, it's not supposed
to be pickleable data. Nor should we be pickling it anyway.

Just put a @memoize on the CreateSvnFileSystemForBranch.

[epeterson, 2013/05/13 02:38:10]

Done.

+      self._object_store.Set('%s%s' % (SVN, version), svn_file_system)
+
+    json_string = svn_file_system.ReadSingle('%s/_permission_features.json'
+                                             % svn_constants.API_PATH)
+    permission_features = json.loads(json_comment_eater.Nom(json_string))

[not at google - send to devlin, 2013/04/30 18:34:19]

same comment re json parsing i.e. use json_parse.Parse (and inline it).

[epeterson, 2013/05/13 02:38:10]

Done.

+
+    existence = 'stable' == self._GetChannelFromPermissionFeatures(
+        api_name, permission_features)
+    self._object_store.Set('%s%s%s' % (PERMISSION, api_name, version),
+                           existence)
+    return existence
+
+  def _CheckManifestExistence(self, api_name, version):
+    '''As a fallback measure for an API not being found in _permission_features,
+    check _manifest_features.json for the given |version| of Chrome,
+    and return a boolean representing whether or not the API was represented
+    within the file.
+    '''
+    #_manifest_features.json uses unix_hacker_style API names.
+    api_name = model.UnixName(api_name)
+    existence = self._object_store.Get('%s%s%s'
+                                       % (MANIFEST, api_name, version)).Get()
+    if existence is not None:
+      return existence
+
+    svn_file_system = self._object_store.Get('%s%s' % (SVN, version)).Get()
+    if svn_file_system is None:
+      svn_file_system = self._CreateSvnFileSystemForBranch(
+          self._chrome_version_utility.GetBranchNumberForVersion(version))
+      self._object_store.Set('%s%s' % (SVN, version), svn_file_system)
+
+    json_string = svn_file_system.ReadSingle('%s/_manifest_features.json'
+                                             % svn_constants.API_PATH)
+    manifest_features = json.loads(json_comment_eater.Nom(json_string))
+    if manifest_features.get(api_name, None) is not None:
+      existence = True
+    else:
+      existence = False
+    self._object_store.Set('%s%s%s' % (MANIFEST, api_name, version), existence)
+    return existence

[not at google - send to devlin, 2013/04/30 18:34:19]

similar comments

[epeterson, 2013/05/13 02:38:10]

Done.

+
+  def _GetAllNames(self, files_dict):
+    '''Returns extension-less names for a list of files.
+    '''
+    files = files_dict[svn_constants.API_PATH + '/']
+    return [model.UnixName(os.path.splitext(f)[0]) for f in files]
+
+  def _CheckFileSystemExistence(self, api_name, version):
+    '''Uses a list of API names for a given branch to check simply for the
+    existence of a given API in that branch.
+    '''
+    names = self._object_store.Get('%s%s' % (FILESYS, version)).Get()
+    if names is not None:
+      return api_name in names
+
+    svn_file_system = self._object_store.Get('%s%s' % (SVN, version)).Get()
+    if svn_file_system is None:
+      svn_file_system = self._CreateSvnFileSystemForBranch(
+          self._chrome_version_utility.GetBranchNumberForVersion(version))
+      self._object_store.Set('%s%s' % (SVN, version), svn_file_system)
+    names = svn_file_system.Read([svn_constants.API_PATH + '/']).Get()
+    names = self._GetAllNames(names)
+    self._object_store.Set('%s%s' % (FILESYS, version), names)
+    return api_name in names

[not at google - send to devlin, 2013/04/30 18:34:19]

similar comments

[epeterson, 2013/05/13 02:38:10]

Done.

+
+  def _FindEarliestStableAvailability(self, api_name, version):
+    '''Searches in descending order through filesystem caches tied to specific
+    chrome version numbers and looks for the availability of an API, |api_name|,
+    on the stable channel. When a version is found where the API is no longer
+    available on stable, returns the previous version number (the last known
+    version where the API was stable).
+    '''
+    # The _permission_features.json file (with 'channel' keys) is present only
+    # in Chrome 20 and onwards.
+    while version >= 20:
+      if ((api_name in self._permission_apis
+          and self._CheckStablePermissionExistence(api_name, str(version)))
+        or (api_name in self._manifest_apis
+            and self._CheckManifestExistence(api_name, str(version)))
+        or (api_name in self._orphan_apis
+            and self._CheckFileSystemExistence(api_name, str(version)))):
+        version -= 1
+      else:
+        break
+    version += 1
+    return str(version)

[not at google - send to devlin, 2013/04/30 18:34:19]

return version + 1 rather than incrementing

[epeterson, 2013/05/13 02:38:10]

Done.

+
+  def GetAvailability(self, api_name):
+    '''Determines the availability for an API by testing several scenarios.
+    (i.e. Is the API experimental? Only available on certain development
+    channels? If it's stable, when did it first become stable? etc.)
+    '''
+    availability = self._object_store.Get(api_name).Get()
+    if availability is not None:
+      return availability
+
+    # Check for a predetermined availability for this API.
+    availability = self._existing_availabilities.get(api_name, None)

[not at google - send to devlin, 2013/04/30 18:34:19]

the None is implicit in get()

[epeterson, 2013/05/13 02:38:10]

Done.

+    if availability is not None:
+      self._object_store.Set(api_name, availability)
+      return availability
+
+    latest_version = str(self._chrome_version_utility.GetLatestVersionNumber())
+    channel = self._GetChannelFromPermissionFeatures(api_name,
+                                                     self._permission_features)
+    if channel is not None:
+       # This is a special case: availability can be any of the dev channels,
+       # to check for this information instead of checking just for existence.
+       availability = channel
+       self._permission_apis.append(api_name)
+    elif self._CheckManifestExistence(api_name, latest_version):
+      availability = 'stable'
+      self._manifest_apis.append(api_name)
+    elif self._CheckFileSystemExistence(api_name, latest_version):
+      availability = 'stable'
+      self._orphan_apis.append(api_name)
+    else:
+      logging.warning('%s could not be located. %s\'s availability is None.'
+                      % (api_name, api_name))
+      availability = None

[not at google - send to devlin, 2013/04/30 18:34:19]

throw a FileNotFoundError here or something, so we notice the failures. If our
data is correct it shouldn't happen, right?

[epeterson, 2013/05/13 02:38:10]

Done.

+
+    if availability != 'stable':
+      self._object_store.Set(api_name, availability)
+      return availability
+
+    availability = self._FindEarliestStableAvailability(api_name,
+                                                        int(latest_version) - 1)
+    self._object_store.Set(api_name, availability)
+    return availability

[not at google - send to devlin, 2013/04/30 18:34:19]

slightly nicer factoring:

if availability == 'stable':
  availability = self._FindEarliest...
self._obejcT_store.Set()
return availability

[epeterson, 2013/05/13 02:38:10]

Done.