Docserver: Use GitilesFileSystem instead of SubversionFileSystem

chrome/common/extensions/docs/server2/gitiles_file_system.py

Index: chrome/common/extensions/docs/server2/gitiles_file_system.py
diff --git a/chrome/common/extensions/docs/server2/gitiles_file_system.py b/chrome/common/extensions/docs/server2/gitiles_file_system.py
index df907e6b08ab83d9783629f476654cda8230aa7e..8470affd80c96e38aeca637c45b413300ad92db7 100644
--- a/chrome/common/extensions/docs/server2/gitiles_file_system.py
+++ b/chrome/common/extensions/docs/server2/gitiles_file_system.py
@@ -2,14 +2,17 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+
 from base64 import b64decode
 from itertools import izip
+import logging
 import json
 import posixpath
+import time
 import traceback
 
 from appengine_url_fetcher import AppEngineUrlFetcher
-from appengine_wrappers import IsDownloadError
+from appengine_wrappers import IsDownloadError, app_identity
 from docs_server_utils import StringIdentity
 from file_system import (FileNotFoundError,
                          FileSystem,
@@ -18,8 +21,9 @@ from file_system import (FileNotFoundError,
 from future import All, Future
 from path_util import AssertIsValid, IsDirectory, ToDirectory
 from third_party.json_schema_compiler.memoize import memoize
-from url_constants import GITILES_BASE, GITILES_BRANCH_BASE
-
+from url_constants import (GITILES_BASE,
+                           GITILES_BRANCH_BASE,
+                           GITILES_OAUTH2_SCOPE)
 
 _JSON_FORMAT = '?format=JSON'
 _TEXT_FORMAT = '?format=TEXT'
@@ -66,9 +70,12 @@ class GitilesFileSystem(FileSystem):
     need to use posixpath.join.
     '''
     AssertIsValid(url)
-    return self._fetcher.FetchAsync('%s/%s' % (self._base_url, url))
+    access_token, _ = app_identity.get_access_token(GITILES_OAUTH2_SCOPE)
+    return self._fetcher.FetchAsync('%s/%s' % (self._base_url, url),
+                                    access_token=access_token)
 
-  def _ResolveFetchContent(self, path, fetch_future, skip_not_found=False):
+  def _ResolveFetchContent(self, path, fetch_future, retry,
+                           skip_not_found=False):
     '''Returns a future to cleanly resolve |fetch_future|.
     '''
     def handle(e):
@@ -84,11 +91,17 @@ class GitilesFileSystem(FileSystem):
           return None
         raise FileNotFoundError('Got 404 when fetching %s for Get from %s' %
                                 (path, self._base_url))
+      if result.status_code == 429:
+        logging.warning('Access throttled when fetching %s for Get from %s' %
+            (path, self._base_url))
+        time.sleep(30)
+        return retry().Then(get_content, handle)
       if result.status_code != 200:
         raise FileSystemError(
             'Got %s when fetching %s for Get from %s, content %s' %
             (result.status_code, path, self._base_url, result.content))
       return result.content
+
     return fetch_future.Then(get_content, handle)
 
   def Read(self, paths, skip_not_found=False):
@@ -118,8 +131,11 @@ class GitilesFileSystem(FileSystem):
       return path + (_JSON_FORMAT if IsDirectory(path) else _TEXT_FORMAT)
 
     # A list of tuples of the form (path, Future).
-    fetches = ((path, self._FetchAsync(fixup_url_format(path)))
-               for path in paths)
+    fetches = []
+    for path in paths:
+      def make_fetch_future():
+        return self._FetchAsync(fixup_url_format(path))
+      fetches.append((path, make_fetch_future(), make_fetch_future))
 
     def parse_contents(results):
       value = {}
@@ -130,8 +146,9 @@ class GitilesFileSystem(FileSystem):
         # http://tools.ietf.org/html/rfc4648 for info about base64).
         value[path] = (list_dir if IsDirectory(path) else b64decode)(content)
       return value
-    return All(self._ResolveFetchContent(path, future, skip_not_found)
-               for path, future in fetches).Then(parse_contents)
+
+    return All(self._ResolveFetchContent(path, future, factory, skip_not_found)
+               for path, future, factory in fetches).Then(parse_contents)
 
   def Refresh(self):
     return Future(value=())
@@ -166,8 +183,15 @@ class GitilesFileSystem(FileSystem):
     # different from '<gitiles_url>/<branch>/?format=JSON': the latter serves
     # the root directory JSON content, whereas the former serves the branch
     # commit info JSON content.
-    fetch_future = self._fetcher.FetchAsync(self._base_url + _JSON_FORMAT)
-    content_future = self._ResolveFetchContent(self._base_url, fetch_future)
+
+    def make_fetch_future():
+      access_token, _ = app_identity.get_access_token(GITILES_OAUTH2_SCOPE)
+      return self._fetcher.FetchAsync(self._base_url + _JSON_FORMAT,
+                                      access_token = access_token)
+
+    fetch_future = make_fetch_future()
+    content_future = self._ResolveFetchContent(self._base_url, fetch_future,
+                                               make_fetch_future)
     return content_future.Then(lambda json: _ParseGitilesJson(json)[key])
 
   def GetCommitID(self):
@@ -175,6 +199,11 @@ class GitilesFileSystem(FileSystem):
     '''
     return self._GetCommitInfo('commit')
 
+  def GetPreviousCommitID(self):
+    '''Returns a future that resolves to the previous commit ID for this branch.
+    '''
+    return self._GetCommitInfo('parents').Then(lambda parents: parents[0])
+
   def StatAsync(self, path):
     dir_, filename = posixpath.split(path)
     def stat(content):
@@ -187,8 +216,13 @@ class GitilesFileSystem(FileSystem):
         raise FileNotFoundError(
             '%s from %s was not in child versions for Stat' % (filename, path))
       return StatInfo(stat_info.child_versions[filename])
-    fetch_future = self._FetchAsync(ToDirectory(dir_) + _JSON_FORMAT)
-    return self._ResolveFetchContent(path, fetch_future).Then(stat)
+
+    def make_fetch_future():
+      return self._FetchAsync(ToDirectory(dir_) + _JSON_FORMAT)
+
+    fetch_future = make_fetch_future()
+    return self._ResolveFetchContent(path, fetch_future,
+                                     make_fetch_future).Then(stat)
 
   def GetIdentity(self):
     # NOTE: Do not use commit information to create the string identity.