Docserver overhaul: Gitiles away from me.

chrome/common/extensions/docs/server2/gcs_file_system.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from third_party.cloudstorage import cloudstorage_api
-from third_party.cloudstorage import common
-from third_party.cloudstorage import errors
+import json
+import logging
+import posixpath
+import traceback
+import urllib
 
 from docs_server_utils import StringIdentity
+from environment_wrappers import CreateUrlFetcher
 from file_system import FileSystem, FileNotFoundError, StatInfo
 from future import Future
 from path_util import (
     AssertIsDirectory, AssertIsFile, AssertIsValid, IsDirectory, Join)
 
-import logging
-import traceback
-
 
 # See gcs_file_system_provider.py for documentation on using Google Cloud
 # Storage as a filesystem.
 #
 # Note that the path requirements for GCS are different for the docserver;
 # GCS requires that paths start with a /, we require that they don't.
 
 
 # Name of the file containing the Git hash of the latest commit sync'ed
 # to Cloud Storage. This file is generated by the Github->GCS sync script
-LAST_COMMIT_HASH_FILENAME = '.__lastcommit.txt'
+_LAST_COMMIT_HASH_FILENAME = '.__lastcommit.txt'
 
-def _ReadFile(filename):
-  AssertIsFile(filename)
-  try:
-    with cloudstorage_api.open('/' + filename, 'r') as f:
-      return f.read()
-  except errors.Error:
-    raise FileNotFoundError('Read failed for %s: %s' % (filename,
-        traceback.format_exc()))
 
-def _ListDir(dir_name, recursive=False):
-  AssertIsDirectory(dir_name)
-  try:
-    # The listbucket method uses a prefix approach to simulate hierarchy.
-    # Calling it with the "delimiter" argument set to '/' gets only files
-    # directly inside the directory, not all recursive content.
-    delimiter = None if recursive else '/'
-    files = cloudstorage_api.listbucket('/' + dir_name, delimiter=delimiter)
-    return [os_path.filename.lstrip('/')[len(dir_name):] for os_path in files]
-  except errors.Error:
-    raise FileNotFoundError('cloudstorage.listbucket failed for %s: %s' %
-                            (dir_name, traceback.format_exc()))
+# Base URL for GCS requests.
+_STORAGE_API_BASE = 'https://www.googleapis.com/storage/v1'
 
-def _CreateStatInfo(bucket, path):
-  full_path = Join(bucket, path)
-  last_commit_file = Join(bucket, LAST_COMMIT_HASH_FILENAME)
-  try:
-    last_commit = _ReadFile(last_commit_file)
-    if IsDirectory(full_path):
-      child_versions = dict((filename, last_commit) 
-                            for filename in _ListDir(full_path)) 
-    else:
-      child_versions = None
-    return StatInfo(last_commit, child_versions)
-  except (TypeError, errors.Error):
-    raise FileNotFoundError('cloudstorage.stat failed for %s: %s' % (path,
-                            traceback.format_exc()))
 
 class CloudStorageFileSystem(FileSystem):
   '''FileSystem implementation which fetches resources from Google Cloud
   Storage.
   '''
-  def __init__(self, bucket, debug_access_token=None, debug_bucket_prefix=None):
+  def __init__(self, bucket, debug_bucket_prefix=None):
     self._bucket = bucket
-    if debug_access_token:
-      logging.debug('gcs: using debug access token: %s' % debug_access_token)
-      common.set_access_token(debug_access_token)
-    if debug_bucket_prefix:
-      logging.debug('gcs: prefixing all bucket names with %s' %
-                    debug_bucket_prefix)
-      self._bucket = debug_bucket_prefix + self._bucket
+    self._access_token = None
+    self._last_commit_hash = None
     AssertIsValid(self._bucket)
 
   def Read(self, paths, skip_not_found=False):
     def resolve():
-      try:
-        result = {}
-        for path in paths:
-          full_path = Join(self._bucket, path)
-          logging.debug('gcs: requested path "%s", reading "%s"' %
-                        (path, full_path))
-          if IsDirectory(path):
-            result[path] = _ListDir(full_path)
-          else:
-            result[path] = _ReadFile(full_path)
-        return result
-      except errors.AuthorizationError:
-        self._warnAboutAuthError()
-        raise
+      result = {}
+      for path in paths:
+        if IsDirectory(path):
+          result[path] = self._ListDir(path)
+        else:
+          result[path] = self._ReadFile(path)
+      return result
 
     return Future(callback=resolve)
 
   def Refresh(self):
     return Future(value=())
 
   def Stat(self, path):
     AssertIsValid(path)
-    try:
-      return _CreateStatInfo(self._bucket, path)
-    except errors.AuthorizationError:
-      self._warnAboutAuthError()
-      raise
+    return self._CreateStatInfo(path)
 
   def GetIdentity(self):
     return '@'.join((self.__class__.__name__, StringIdentity(self._bucket)))
 
+  def _CreateStatInfo(self, path):

[not at google - send to devlin, 2015/06/04 22:40:45]

What actually changed in this file?

[Ken Rockot(use gerrit already), 2015/06/05 00:21:50]

It's using the GCS JSON API directly now instead of the (appengine-only)
cloudstorage_api module. It also assumes no need for access tokens since all the
buckets are public.

+    if not self._last_commit_hash:
+      self._last_commit_hash = self._ReadFile(_LAST_COMMIT_HASH_FILENAME)
+    if IsDirectory(path):
+      child_versions = dict((filename, self._last_commit_hash)
+                            for filename in self._ListDir(path))
+    else:
+      child_versions = None
+    return StatInfo(self._last_commit_hash, child_versions)
+
+  def _ReadFile(self, path):
+    AssertIsFile(path)
+    return self._FetchObjectData(path)
+
+  def _ListDir(self, path, recursive=False):
+    AssertIsDirectory(path)
+    # The listbucket method uses a prefix approach to simulate hierarchy.
+    # Calling it with the "delimiter" argument set to '/' gets only files
+    # directly inside the directory, not all recursive content.
+
+    # Subdirectories are returned in the 'prefixes' property, but they are
+    # full paths from the root. This plucks off the name of the leaf with a
+    # trailing slash.
+    def path_from_prefix(prefix):
+      return posixpath.split(posixpath.split(prefix)[0])[1] + '/'
+
+    query = { 'prefix': path }
+    if not recursive:
+      query['delimiter'] = '/'
+    root_object = json.loads(self._FetchObject('', query=query))
+    files = [posixpath.basename(o['name'])
+             for o in root_object.get('items', [])]
+    dirs = [path_from_prefix(prefix)
+            for prefix in root_object.get('prefixes', [])]
+    return files + dirs
+
+  def _FetchObject(self, path, query={}):
+    # Escape the path, including slashes.
+    url_path = urllib.quote(path.lstrip('/'), safe='')
+    fetcher = CreateUrlFetcher()
+    object_url = '%s/b/%s/o/%s' % (_STORAGE_API_BASE, self._bucket, url_path)
+    response = fetcher.Fetch(object_url, query=query)
+    if response.status_code != 200:
+      raise FileNotFoundError(
+          'Path %s not found in GCS bucket %s' % (path, self._bucket))
+    return response.content
+
+  def _FetchObjectData(self, path, query={}):
+    q = query.copy()
+    q.update({ 'alt': 'media' })
+    return self._FetchObject(path, query=q)
+
   def __repr__(self):
     return 'CloudStorageFileSystem(%s)' % self._bucket
-
-  def _warnAboutAuthError(self):
-    logging.warn(('Authentication error on Cloud Storage. Check if your'
-                  ' appengine project has permissions to Read the GCS'
-                  ' buckets. If you are running a local appengine server,'
-                  ' you need to set an access_token in'
-                  ' local_debug/gcs_debug.conf.'
-                  ' Remember that this token expires in less than 10'
-                  ' minutes, so keep it updated. See'
-                  ' gcs_file_system_provider.py for instructions.'));
-    logging.debug(traceback.format_exc())