make gs_utils.py work without a .boto credential file

py/utils/gs_utils.py

Index: py/utils/gs_utils.py
diff --git a/py/utils/gs_utils.py b/py/utils/gs_utils.py
index afc374841b3d7ca5f56fdd2e7848a85b5a4e5110..53cafbe3d38853dc42746e2fdda43ecf45a32604 100755
--- a/py/utils/gs_utils.py
+++ b/py/utils/gs_utils.py
@@ -37,9 +37,11 @@ for import_subdir in ['boto']:
     # imported versions are favored over others that might be in the path.
     sys.path.insert(0, import_dirpath)
 from boto.gs import acl
+from boto.gs.bucket import Bucket
 from boto.gs.connection import GSConnection
 from boto.gs.key import Key
 from boto.s3.bucketlistresultset import BucketListResultSet
+from boto.s3.connection import SubdomainCallingFormat
 from boto.s3.prefix import Prefix
 
 # Permissions that may be set on each file in Google Storage.
@@ -67,24 +69,38 @@ FIELD_BY_ID_TYPE = {
 }
 
 
+class AnonymousGSConnection(GSConnection):
+  """The GSConnection class in the boto library doesn't allow for anonymous
+  connections (connection without credentials).
+  """
+  def __init__(self):
+    super(GSConnection, self).__init__(
+        anon=True, host=GSConnection.DefaultHost,
+        calling_format=SubdomainCallingFormat(), provider='google',

[rmistry, 2014/07/16 13:34:21]

Could you explain the provider='google' ?

[epoger, 2014/07/16 13:47:42]

Good question.  I have tried to do so in patchset 5...

+        bucket_class=Bucket)
+
+
 class GSUtils(object):
   """Utilities for accessing Google Cloud Storage, using the boto library."""
 
-  def __init__(self, boto_file_path=os.path.join('~','.boto')):
+  def __init__(self, boto_file_path=None):
     """Constructor.
 
     Params:
       boto_file_path: full path (local-OS-style) on local disk where .boto
-          credentials file can be found.  An exception is thrown if this file
-          is missing.
-          TODO(epoger): Change missing-file behavior: allow the caller to
-          operate on public files in Google Storage.
+          credentials file can be found.  If None, then the GSUtils object
+          created will be able to access only public files in Google Storage.
+
+    Raises an exception if no file is found at boto_file_path, or if the file
+    found there is malformed.
     """
-    boto_file_path = os.path.expanduser(boto_file_path)
-    print 'Reading boto file from %s' % boto_file_path
-    boto_dict = _config_file_as_dict(filepath=boto_file_path)
-    self._gs_access_key_id = boto_dict['gs_access_key_id']
-    self._gs_secret_access_key = boto_dict['gs_secret_access_key']
+    self._gs_access_key_id = None
+    self._gs_secret_access_key = None
+    if boto_file_path:
+      print 'Reading boto file from %s' % boto_file_path
+      boto_dict = _config_file_as_dict(filepath=boto_file_path)
+      self._gs_access_key_id = boto_dict['gs_access_key_id']
+      self._gs_secret_access_key = boto_dict['gs_secret_access_key']
 
   def delete_file(self, bucket, path):
     """Delete a single file within a GS bucket.
@@ -258,10 +274,12 @@ class GSUtils(object):
 
   def _create_connection(self):
     """Returns a GSConnection object we can use to access Google Storage."""
-    return GSConnection(
-        gs_access_key_id=self._gs_access_key_id,
-        gs_secret_access_key=self._gs_secret_access_key)
-
+    if self._gs_access_key_id:
+      return GSConnection(
+          gs_access_key_id=self._gs_access_key_id,
+          gs_secret_access_key=self._gs_secret_access_key)
+    else:
+      return AnonymousGSConnection()
 
 def _config_file_as_dict(filepath):
   """Reads a boto-style config file into a dict.
@@ -301,12 +319,27 @@ def _makedirs_if_needed(path):
       raise
 
 
-def _run_self_test():
+def _test_public_read():
+  """Make sure we can read from public files without .boto file credentials."""
+  gs = GSUtils()
+  gs.list_bucket_contents(bucket='chromium-skia-gm-summaries', subdir=None)
+
+
+def _test_authenticated_round_trip():
+  try:
+    gs = GSUtils(boto_file_path=os.path.expanduser(os.path.join('~','.boto')))
+  except:
+    print """
+Failed to instantiate GSUtils object with default .boto file path.
+Do you have a ~/.boto file that provides the credentials needed to read
+and write gs://chromium-skia-gm ?
+"""
+    raise
+
   bucket = 'chromium-skia-gm'
   remote_dir = 'gs_utils_test/%d' % random.randint(0, sys.maxint)
   subdir = 'subdir'
   filenames_to_upload = ['file1', 'file2']
-  gs = GSUtils()
 
   # Upload test files to Google Storage.
   local_src_dir = tempfile.mkdtemp()
@@ -393,11 +426,10 @@ def _run_self_test():
   assert files == [], '%s == []' % files
 
 
-# TODO(epoger): How should we exercise this self-test?
-# I avoided using the standard unittest framework, because these Google Storage
-# operations are expensive and require .boto permissions.
-#
-# How can we automatically test this code without wasting too many resources
-# or needing .boto permissions?
+# TODO(epoger): How should we exercise these self-tests?
+# See http://skbug.com/2751
 if __name__ == '__main__':
-  _run_self_test()
+  _test_public_read()
+  _test_authenticated_round_trip()
+  # TODO(epoger): Add _test_unauthenticated_access() to make sure we raise
+  # an exception when we try to access without needed credentials.