Docserver overhaul: Gitiles away from me.

chrome/common/extensions/docs/server2/gitiles_file_system.py

-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be

[Ken Rockot(use gerrit already), 2015/05/26 00:26:22]

good riddance

[not at google - send to devlin, 2015/06/04 22:40:45]

:-)

-# found in the LICENSE file.
-
-
-from base64 import b64decode
-from itertools import izip
-import json
-import logging
-import posixpath
-import time
-import traceback
-
-from appengine_url_fetcher import AppEngineUrlFetcher
-from appengine_wrappers import IsDownloadError, app_identity
-from docs_server_utils import StringIdentity
-from environment import IsDevServer
-from file_system import (FileNotFoundError,
-                         FileSystem,
-                         FileSystemError,
-                         FileSystemThrottledError,
-                         StatInfo)
-from future import All, Future
-from path_util import AssertIsValid, IsDirectory, ToDirectory
-from third_party.json_schema_compiler.memoize import memoize
-from url_constants import (GITILES_BASE,
-                           GITILES_SRC_ROOT,
-                           GITILES_BRANCHES_PATH,
-                           GITILES_OAUTH2_SCOPE)
-
-
-_JSON_FORMAT = '?format=JSON'
-_TEXT_FORMAT = '?format=TEXT'
-_AUTH_PATH_PREFIX = '/a'
-
-
-def _ParseGitilesJson(json_data):
-  '''json.loads with fix-up for non-executable JSON. Use this to parse any JSON
-  data coming from Gitiles views.
-  '''
-  return json.loads(json_data[json_data.find('{'):])
-
-
-def _CreateStatInfo(json_data):
-  '''Returns a StatInfo object comprised of the tree ID for |json_data|,
-  as well as the tree IDs for the entries in |json_data|.
-  '''
-  tree = _ParseGitilesJson(json_data)
-  return StatInfo(tree['id'],
-                  dict((e['name'], e['id']) for e in tree['entries']))
-
-
-class GitilesFileSystem(FileSystem):
-  '''Class to fetch filesystem data from the Chromium project's gitiles
-  service.
-  '''
-  _logged_tokens = set()
-
-  @classmethod
-  def Create(cls, branch='master', commit=None):
-    token, _ = app_identity.get_access_token(GITILES_OAUTH2_SCOPE)
-
-    # Log the access token (once per token) so that it can be sneakily re-used
-    # in development.
-    if token not in cls._logged_tokens:
-      logging.info('Got token %s for scope %s' % (token, GITILES_OAUTH2_SCOPE))
-      cls._logged_tokens.add(token)
-
-    # Only include forced-auth (/a/) in the Gitiles URL if we have a token and
-    # this is not the development server.
-    path_prefix = ('' if token is None or IsDevServer()
-                      else _AUTH_PATH_PREFIX)
-    if commit:
-      base_url = '%s%s/%s/%s' % (
-          GITILES_BASE, path_prefix, GITILES_SRC_ROOT, commit)
-    elif branch is 'master':
-      base_url = '%s%s/%s/master' % (
-          GITILES_BASE, path_prefix, GITILES_SRC_ROOT)
-    else:
-      base_url = '%s%s/%s/%s/%s' % (
-          GITILES_BASE, path_prefix, GITILES_SRC_ROOT,
-          GITILES_BRANCHES_PATH, branch)
-    return GitilesFileSystem(AppEngineUrlFetcher(), base_url, branch, commit)
-
-  def __init__(self, fetcher, base_url, branch, commit):
-    self._fetcher = fetcher
-    self._base_url = base_url
-    self._branch = branch
-    self._commit = commit
-
-  def _FetchAsync(self, url):
-    '''Convenience wrapper for fetcher.FetchAsync, so callers don't
-    need to use posixpath.join.
-    '''
-    AssertIsValid(url)
-    access_token, _ = app_identity.get_access_token(GITILES_OAUTH2_SCOPE)
-    return self._fetcher.FetchAsync('%s/%s' % (self._base_url, url),
-                                    access_token=access_token)
-
-  def _ResolveFetchContent(self, path, fetch_future, skip_not_found=False):
-    '''Returns a future to cleanly resolve |fetch_future|.
-    '''
-    def handle(e):
-      if skip_not_found and IsDownloadError(e):
-        return None
-      exc_type = FileNotFoundError if IsDownloadError(e) else FileSystemError
-      raise exc_type('%s fetching %s for Get from %s: %s' %
-          (type(e).__name__, path, self._base_url, traceback.format_exc()))
-
-    def get_content(result):
-      if result.status_code == 404:
-        if skip_not_found:
-          return None
-        raise FileNotFoundError('Got 404 when fetching %s for Get from %s' %
-                                (path, self._base_url))
-      if result.status_code == 429:
-        logging.warning('Access throttled when fetching %s for Get from %s' %
-            (path, self._base_url))
-        raise FileSystemThrottledError(
-            'Access throttled when fetching %s for Get from %s' %
-            (path, self._base_url))
-      if result.status_code != 200:
-        raise FileSystemError(
-            'Got %s when fetching %s for Get from %s, content %s' %
-            (result.status_code, path, self._base_url, result.content))
-      return result.content
-
-    return fetch_future.Then(get_content, handle)
-
-  def Read(self, paths, skip_not_found=False):
-    # Directory content is formatted in JSON in Gitiles as follows:
-    #
-    #   {
-    #     "id": "12a5464de48d2c46bc0b2dc78fafed75aab554fa", # The tree ID.
-    #     "entries": [
-    #       {
-    #         "mode": 33188,
-    #         "type": "blob",
-    #           "id": "ab971ca447bc4bce415ed4498369e00164d91cb6", # File ID.
-    #         "name": ".gitignore"
-    #       },
-    #       ...
-    #     ]
-    #   }
-    def list_dir(json_data):
-      entries = _ParseGitilesJson(json_data).get('entries', [])
-      return [e['name'] + ('/' if e['type'] == 'tree' else '') for e in entries]
-
-    def fixup_url_format(path):
-      # By default, Gitiles URLs display resources in HTML. To get resources
-      # suitable for our consumption, a '?format=' string must be appended to
-      # the URL. The format may be one of 'JSON' or 'TEXT' for directory or
-      # text resources, respectively.
-      return path + (_JSON_FORMAT if IsDirectory(path) else _TEXT_FORMAT)
-
-    # A list of tuples of the form (path, Future).
-    fetches = [(path, self._FetchAsync(fixup_url_format(path)))
-               for path in paths]
-
-    def parse_contents(results):
-      value = {}
-      for path, content in izip(paths, results):
-        if content is None:
-          continue
-        # Gitiles encodes text content in base64 (see
-        # http://tools.ietf.org/html/rfc4648 for info about base64).
-        value[path] = (list_dir if IsDirectory(path) else b64decode)(content)
-      return value
-
-    return All(self._ResolveFetchContent(path, future, skip_not_found)
-               for path, future in fetches).Then(parse_contents)
-
-  def Refresh(self):
-    return Future(value=())
-
-  @memoize
-  def _GetCommitInfo(self, key):
-    '''Gets the commit information specified by |key|.
-
-    The JSON view for commit info looks like:
-      {
-        "commit": "8fd578e1a7b142cd10a4387861f05fb9459b69e2", # Commit ID.
-        "tree": "3ade65d8a91eadd009a6c9feea8f87db2c528a53",   # Tree ID.
-        "parents": [
-          "a477c787fe847ae0482329f69b39ce0fde047359" # Previous commit ID.
-        ],
-        "author": {
-          "name": "...",
-          "email": "...",
-          "time": "Tue Aug 12 17:17:21 2014"
-        },
-        "committer": {
-          "name": "...",
-          "email": "...",
-          "time": "Tue Aug 12 17:18:28 2014"
-        },
-        "message": "...",
-        "tree_diff": [...]
-      }
-    '''
-    # Commit information for a branch is obtained by appending '?format=JSON'
-    # to the branch URL. Note that '<gitiles_url>/<branch>?format=JSON' is
-    # different from '<gitiles_url>/<branch>/?format=JSON': the latter serves
-    # the root directory JSON content, whereas the former serves the branch
-    # commit info JSON content.
-
-    access_token, _ = app_identity.get_access_token(GITILES_OAUTH2_SCOPE)
-    fetch_future = self._fetcher.FetchAsync(self._base_url + _JSON_FORMAT,
-                                            access_token=access_token)
-    content_future = self._ResolveFetchContent(self._base_url, fetch_future)
-    return content_future.Then(lambda json: _ParseGitilesJson(json)[key])
-
-  def GetCommitID(self):
-    '''Returns a future that resolves to the commit ID for this branch.
-    '''
-    return self._GetCommitInfo('commit')
-
-  def GetPreviousCommitID(self):
-    '''Returns a future that resolves to the previous commit ID for this branch.
-    '''
-    return self._GetCommitInfo('parents').Then(lambda parents: parents[0])
-
-  def StatAsync(self, path):
-    dir_, filename = posixpath.split(path)
-    def stat(content):
-      stat_info = _CreateStatInfo(content)
-      if stat_info.version is None:
-        raise FileSystemError('Failed to find version of dir %s' % dir_)
-      if IsDirectory(path):
-        return stat_info
-      if filename not in stat_info.child_versions:
-        raise FileNotFoundError(
-            '%s from %s was not in child versions for Stat' % (filename, path))
-      return StatInfo(stat_info.child_versions[filename])
-
-    fetch_future = self._FetchAsync(ToDirectory(dir_) + _JSON_FORMAT)
-    return self._ResolveFetchContent(path, fetch_future).Then(stat)
-
-  def GetIdentity(self):
-    if self._branch == 'master':
-      # A master FS always carries the same identity even if pinned to a commit.
-      str_id = 'master'
-    elif self._commit is not None:
-      str_id = self._commit
-    else:
-      str_id = '%s/%s' % (GITILES_BRANCHES_PATH, self._branch)
-    return '@'.join((self.__class__.__name__, StringIdentity(
-        '%s/%s/%s' % (GITILES_BASE, GITILES_SRC_ROOT, str_id))))
-
-  def GetVersion(self):
-    return self._commit