Docserver overhaul: Gitiles away from me.

chrome/common/extensions/docs/server2/gcs_file_system.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from third_party.cloudstorage import cloudstorage_api
-from third_party.cloudstorage import common
-from third_party.cloudstorage import errors
+import json
+import logging
+import posixpath
+import traceback
+import urllib
 
 from docs_server_utils import StringIdentity
+from environment_wrappers import CreateUrlFetcher

[Ken Rockot(use gerrit already), 2015/05/26 00:26:24]

Unfortunately cloudstorage_api only works from AppEngine and we needed GCS
support in other environments.

So I reimplemented CloudStorageFileSystem using the Cloud Storage JSON API
directly on top of whatever UrlFetcher the environment provides. This was pretty
trivial since we only need listing and file reading.

 from file_system import FileSystem, FileNotFoundError, StatInfo
 from future import Future
 from path_util import (
     AssertIsDirectory, AssertIsFile, AssertIsValid, IsDirectory, Join)
 
-import logging
-import traceback
-
 
 # See gcs_file_system_provider.py for documentation on using Google Cloud
 # Storage as a filesystem.
 #
 # Note that the path requirements for GCS are different for the docserver;
 # GCS requires that paths start with a /, we require that they don't.
 
 
 # Name of the file containing the Git hash of the latest commit sync'ed
 # to Cloud Storage. This file is generated by the Github->GCS sync script
-LAST_COMMIT_HASH_FILENAME = '.__lastcommit.txt'
+_LAST_COMMIT_HASH_FILENAME = '.__lastcommit.txt'
 
-def _ReadFile(filename):
-  AssertIsFile(filename)
-  try:
-    with cloudstorage_api.open('/' + filename, 'r') as f:
-      return f.read()
-  except errors.Error:
-    raise FileNotFoundError('Read failed for %s: %s' % (filename,
-        traceback.format_exc()))
 
-def _ListDir(dir_name, recursive=False):
-  AssertIsDirectory(dir_name)
-  try:
-    # The listbucket method uses a prefix approach to simulate hierarchy.
-    # Calling it with the "delimiter" argument set to '/' gets only files
-    # directly inside the directory, not all recursive content.
-    delimiter = None if recursive else '/'
-    files = cloudstorage_api.listbucket('/' + dir_name, delimiter=delimiter)
-    return [os_path.filename.lstrip('/')[len(dir_name):] for os_path in files]
-  except errors.Error:
-    raise FileNotFoundError('cloudstorage.listbucket failed for %s: %s' %
-                            (dir_name, traceback.format_exc()))
+# Base URL for GCS requests.
+_STORAGE_API_BASE = 'https://www.googleapis.com/storage/v1'
 
-def _CreateStatInfo(bucket, path):
-  full_path = Join(bucket, path)
-  last_commit_file = Join(bucket, LAST_COMMIT_HASH_FILENAME)
-  try:
-    last_commit = _ReadFile(last_commit_file)
-    if IsDirectory(full_path):
-      child_versions = dict((filename, last_commit) 
-                            for filename in _ListDir(full_path)) 
-    else:
-      child_versions = None
-    return StatInfo(last_commit, child_versions)
-  except (TypeError, errors.Error):
-    raise FileNotFoundError('cloudstorage.stat failed for %s: %s' % (path,
-                            traceback.format_exc()))
 
 class CloudStorageFileSystem(FileSystem):
   '''FileSystem implementation which fetches resources from Google Cloud
   Storage.
   '''
-  def __init__(self, bucket, debug_access_token=None, debug_bucket_prefix=None):
+  def __init__(self, bucket, debug_bucket_prefix=None):
     self._bucket = bucket
-    if debug_access_token:
-      logging.debug('gcs: using debug access token: %s' % debug_access_token)
-      common.set_access_token(debug_access_token)
-    if debug_bucket_prefix:
-      logging.debug('gcs: prefixing all bucket names with %s' %
-                    debug_bucket_prefix)
-      self._bucket = debug_bucket_prefix + self._bucket
+    self._access_token = None
+    self._last_commit_hash = None
     AssertIsValid(self._bucket)
 
   def Read(self, paths, skip_not_found=False):
     def resolve():
-      try:
-        result = {}
-        for path in paths:
-          full_path = Join(self._bucket, path)
-          logging.debug('gcs: requested path "%s", reading "%s"' %
-                        (path, full_path))
-          if IsDirectory(path):
-            result[path] = _ListDir(full_path)
-          else:
-            result[path] = _ReadFile(full_path)
-        return result
-      except errors.AuthorizationError:
-        self._warnAboutAuthError()
-        raise
+      result = {}
+      for path in paths:
+        if IsDirectory(path):
+          result[path] = self._ListDir(path)
+        else:
+          result[path] = self._ReadFile(path)
+      return result
 
     return Future(callback=resolve)
 
   def Refresh(self):
     return Future(value=())
 
   def Stat(self, path):
     AssertIsValid(path)
-    try:
-      return _CreateStatInfo(self._bucket, path)
-    except errors.AuthorizationError:
-      self._warnAboutAuthError()
-      raise
+    return self._CreateStatInfo(path)
 
   def GetIdentity(self):
     return '@'.join((self.__class__.__name__, StringIdentity(self._bucket)))
 
+  def _CreateStatInfo(self, path):
+    if not self._last_commit_hash:
+      self._last_commit_hash = self._ReadFile(_LAST_COMMIT_HASH_FILENAME)
+    if IsDirectory(path):
+      child_versions = dict((filename, self._last_commit_hash)
+                            for filename in self._ListDir(path))
+    else:
+      child_versions = None
+    return StatInfo(self._last_commit_hash, child_versions)
+
+  def _ReadFile(self, path):
+    AssertIsFile(path)
+    return self._FetchObjectData(path)
+
+  def _ListDir(self, path, recursive=False):
+    AssertIsDirectory(path)
+    # The listbucket method uses a prefix approach to simulate hierarchy.
+    # Calling it with the "delimiter" argument set to '/' gets only files
+    # directly inside the directory, not all recursive content.
+
+    # Subdirectories are returned in the 'prefixes' property, but they are
+    # full paths from the root. This plucks off the name of the leaf with a
+    # trailing slash.
+    def path_from_prefix(prefix):
+      return posixpath.split(posixpath.split(prefix)[0])[1] + '/'
+
+    query = { 'prefix': path }
+    if not recursive:
+      query['delimiter'] = '/'
+    root_object = json.loads(self._FetchObject('', query=query))
+    files = [posixpath.basename(o['name'])
+             for o in root_object.get('items', [])]
+    dirs = [path_from_prefix(prefix)
+            for prefix in root_object.get('prefixes', [])]
+    return files + dirs
+
+  def _FetchObject(self, path, query={}):
+    # Escape the path, including slashes.
+    url_path = urllib.quote(path.lstrip('/'), safe='')
+    fetcher = CreateUrlFetcher()
+    object_url = '%s/b/%s/o/%s' % (_STORAGE_API_BASE, self._bucket, url_path)
+    response = fetcher.Fetch(object_url, query=query)
+    if response.status_code != 200:
+      raise FileNotFoundError(
+          'Path %s not found in GCS bucket %s' % (path, self._bucket))
+    return response.content
+
+  def _FetchObjectData(self, path, query={}):
+    q = query.copy()
+    q.update({ 'alt': 'media' })
+    return self._FetchObject(path, query=q)
+
   def __repr__(self):
     return 'CloudStorageFileSystem(%s)' % self._bucket
-
-  def _warnAboutAuthError(self):

[Ken Rockot(use gerrit already), 2015/05/26 00:26:24]

Auth is no longer required. The github repos are public, so I made the buckets
public too. I think we can just avoid hosting docserver content in non-public
buckets.

-    logging.warn(('Authentication error on Cloud Storage. Check if your'
-                  ' appengine project has permissions to Read the GCS'
-                  ' buckets. If you are running a local appengine server,'
-                  ' you need to set an access_token in'
-                  ' local_debug/gcs_debug.conf.'
-                  ' Remember that this token expires in less than 10'
-                  ' minutes, so keep it updated. See'
-                  ' gcs_file_system_provider.py for instructions.'));
-    logging.debug(traceback.format_exc())