make gs_utils.py work without a .boto credential file

py/utils/gs_utils.py

 #!/usr/bin/python
 
 # pylint: disable=C0301
 """
 Copyright 2014 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Utilities for accessing Google Cloud Storage, using the boto library (wrapper
@@ skipping 19 matching lines @@
 TRUNK_DIRECTORY = os.path.abspath(os.path.join(
     os.path.dirname(__file__), os.pardir, os.pardir))
 for import_subdir in ['boto']:
   import_dirpath = os.path.join(
       TRUNK_DIRECTORY, 'third_party', 'externals', import_subdir)
   if import_dirpath not in sys.path:
     # We need to insert at the beginning of the path, to make sure that our
     # imported versions are favored over others that might be in the path.
     sys.path.insert(0, import_dirpath)
 from boto.gs import acl
+from boto.gs.bucket import Bucket
 from boto.gs.connection import GSConnection
 from boto.gs.key import Key
 from boto.s3.bucketlistresultset import BucketListResultSet
+from boto.s3.connection import SubdomainCallingFormat
 from boto.s3.prefix import Prefix
 
 # Permissions that may be set on each file in Google Storage.
 # See SupportedPermissions in
 # https://github.com/boto/boto/blob/develop/boto/gs/acl.py
 PERMISSION_NONE  = None
 PERMISSION_OWNER = 'FULL_CONTROL'
 PERMISSION_READ  = 'READ'
 PERMISSION_WRITE = 'WRITE'
 
 # Types of identifiers we can use to set ACLs.
 ID_TYPE_GROUP_BY_DOMAIN = acl.GROUP_BY_DOMAIN
 ID_TYPE_GROUP_BY_EMAIL = acl.GROUP_BY_EMAIL
 ID_TYPE_GROUP_BY_ID = acl.GROUP_BY_ID
 ID_TYPE_USER_BY_EMAIL = acl.USER_BY_EMAIL
 ID_TYPE_USER_BY_ID = acl.USER_BY_ID
 
 # Which field we get/set in ACL entries, depending on ID_TYPE.
 FIELD_BY_ID_TYPE = {
     ID_TYPE_GROUP_BY_DOMAIN: 'domain',
     ID_TYPE_GROUP_BY_EMAIL: 'email_address',
     ID_TYPE_GROUP_BY_ID: 'id',
     ID_TYPE_USER_BY_EMAIL: 'email_address',
     ID_TYPE_USER_BY_ID: 'id',
 }
 
 
+class AnonymousGSConnection(GSConnection):
+  """The GSConnection class in the boto library doesn't allow for anonymous
+  connections (connection without credentials).
+  """
+  def __init__(self):
+    super(GSConnection, self).__init__(
+        anon=True, host=GSConnection.DefaultHost,
+        calling_format=SubdomainCallingFormat(), provider='google',

[rmistry, 2014/07/16 13:34:21]

Could you explain the provider='google' ?

[epoger, 2014/07/16 13:47:42]

Good question.  I have tried to do so in patchset 5...

+        bucket_class=Bucket)
+
+
 class GSUtils(object):
   """Utilities for accessing Google Cloud Storage, using the boto library."""
 
-  def __init__(self, boto_file_path=os.path.join('~','.boto')):
+  def __init__(self, boto_file_path=None):
     """Constructor.
 
     Params:
       boto_file_path: full path (local-OS-style) on local disk where .boto
-          credentials file can be found.  An exception is thrown if this file
-          is missing.
-          TODO(epoger): Change missing-file behavior: allow the caller to
-          operate on public files in Google Storage.
+          credentials file can be found.  If None, then the GSUtils object
+          created will be able to access only public files in Google Storage.
+
+    Raises an exception if no file is found at boto_file_path, or if the file
+    found there is malformed.
     """
-    boto_file_path = os.path.expanduser(boto_file_path)
-    print 'Reading boto file from %s' % boto_file_path
-    boto_dict = _config_file_as_dict(filepath=boto_file_path)
-    self._gs_access_key_id = boto_dict['gs_access_key_id']
-    self._gs_secret_access_key = boto_dict['gs_secret_access_key']
+    self._gs_access_key_id = None
+    self._gs_secret_access_key = None
+    if boto_file_path:
+      print 'Reading boto file from %s' % boto_file_path
+      boto_dict = _config_file_as_dict(filepath=boto_file_path)
+      self._gs_access_key_id = boto_dict['gs_access_key_id']
+      self._gs_secret_access_key = boto_dict['gs_secret_access_key']
 
   def delete_file(self, bucket, path):
     """Delete a single file within a GS bucket.
 
     TODO(epoger): what if bucket or path does not exist?  Should probably raise
     an exception.  Implement, and add a test to exercise this.
 
     Params:
       bucket: GS bucket to delete a file from
       path: full path (Posix-style) of the file within the bucket to delete
@@ skipping 153 matching lines @@
     for item in lister:
       t = type(item)
       if t is Key:
         files.append(item.key[prefix_length:])
       elif t is Prefix:
         dirs.append(item.name[prefix_length:-1])
     return (dirs, files)
 
   def _create_connection(self):
     """Returns a GSConnection object we can use to access Google Storage."""
-    return GSConnection(
-        gs_access_key_id=self._gs_access_key_id,
-        gs_secret_access_key=self._gs_secret_access_key)
-
+    if self._gs_access_key_id:
+      return GSConnection(
+          gs_access_key_id=self._gs_access_key_id,
+          gs_secret_access_key=self._gs_secret_access_key)
+    else:
+      return AnonymousGSConnection()
 
 def _config_file_as_dict(filepath):
   """Reads a boto-style config file into a dict.
 
   Parses all lines from the file of this form: key = value
   TODO(epoger): Create unittest.
 
   Params:
     filepath: path to config file on local disk
 
@@ skipping 19 matching lines @@
   Args:
     path: full path of directory to create
   """
   try:
     os.makedirs(path)
   except OSError as e:
     if e.errno != errno.EEXIST:
       raise
 
 
-def _run_self_test():
+def _test_public_read():
+  """Make sure we can read from public files without .boto file credentials."""
+  gs = GSUtils()
+  gs.list_bucket_contents(bucket='chromium-skia-gm-summaries', subdir=None)
+
+
+def _test_authenticated_round_trip():
+  try:
+    gs = GSUtils(boto_file_path=os.path.expanduser(os.path.join('~','.boto')))
+  except:
+    print """
+Failed to instantiate GSUtils object with default .boto file path.
+Do you have a ~/.boto file that provides the credentials needed to read
+and write gs://chromium-skia-gm ?
+"""
+    raise
+
   bucket = 'chromium-skia-gm'
   remote_dir = 'gs_utils_test/%d' % random.randint(0, sys.maxint)
   subdir = 'subdir'
   filenames_to_upload = ['file1', 'file2']
-  gs = GSUtils()
 
   # Upload test files to Google Storage.
   local_src_dir = tempfile.mkdtemp()
   os.mkdir(os.path.join(local_src_dir, subdir))
   try:
     for filename in filenames_to_upload:
       with open(os.path.join(local_src_dir, subdir, filename), 'w') as f:
         f.write('contents of %s\n' % filename)
       gs.upload_file(source_path=os.path.join(local_src_dir, subdir, filename),
                      dest_bucket=bucket,
@@ skipping 66 matching lines @@
     gs.delete_file(bucket=bucket,
                    path=posixpath.join(remote_dir, subdir, filename))
 
   # Confirm that we deleted all the files we uploaded to Google Storage.
   (dirs, files) = gs.list_bucket_contents(
       bucket=bucket, subdir=posixpath.join(remote_dir, subdir))
   assert dirs == [], '%s == []' % dirs
   assert files == [], '%s == []' % files
 
 
-# TODO(epoger): How should we exercise this self-test?
-# I avoided using the standard unittest framework, because these Google Storage
-# operations are expensive and require .boto permissions.
-#
-# How can we automatically test this code without wasting too many resources
-# or needing .boto permissions?
+# TODO(epoger): How should we exercise these self-tests?
+# See http://skbug.com/2751
 if __name__ == '__main__':
-  _run_self_test()
+  _test_public_read()
+  _test_authenticated_round_trip()
+  # TODO(epoger): Add _test_unauthenticated_access() to make sure we raise
+  # an exception when we try to access without needed credentials.