make gs_utils.py work without a .boto credential file

py/utils/gs_utils.py

 #!/usr/bin/python
 
 # pylint: disable=C0301
 """
 Copyright 2014 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Utilities for accessing Google Cloud Storage, using the boto library (wrapper
@@ skipping 19 matching lines @@
 TRUNK_DIRECTORY = os.path.abspath(os.path.join(
     os.path.dirname(__file__), os.pardir, os.pardir))
 for import_subdir in ['boto']:
   import_dirpath = os.path.join(
       TRUNK_DIRECTORY, 'third_party', 'externals', import_subdir)
   if import_dirpath not in sys.path:
     # We need to insert at the beginning of the path, to make sure that our
     # imported versions are favored over others that might be in the path.
     sys.path.insert(0, import_dirpath)
 from boto.gs import acl
+from boto.gs.bucket import Bucket
 from boto.gs.connection import GSConnection
 from boto.gs.key import Key
 from boto.s3.bucketlistresultset import BucketListResultSet
+from boto.s3.connection import SubdomainCallingFormat
 from boto.s3.prefix import Prefix
 
 # Permissions that may be set on each file in Google Storage.
 # See SupportedPermissions in
 # https://github.com/boto/boto/blob/develop/boto/gs/acl.py
 PERMISSION_NONE  = None
 PERMISSION_OWNER = 'FULL_CONTROL'
 PERMISSION_READ  = 'READ'
 PERMISSION_WRITE = 'WRITE'
 
 # Types of identifiers we can use to set ACLs.
 ID_TYPE_GROUP_BY_DOMAIN = acl.GROUP_BY_DOMAIN
 ID_TYPE_GROUP_BY_EMAIL = acl.GROUP_BY_EMAIL
 ID_TYPE_GROUP_BY_ID = acl.GROUP_BY_ID
 ID_TYPE_USER_BY_EMAIL = acl.USER_BY_EMAIL
 ID_TYPE_USER_BY_ID = acl.USER_BY_ID
 
 # Which field we get/set in ACL entries, depending on ID_TYPE.
 FIELD_BY_ID_TYPE = {
     ID_TYPE_GROUP_BY_DOMAIN: 'domain',
     ID_TYPE_GROUP_BY_EMAIL: 'email_address',
     ID_TYPE_GROUP_BY_ID: 'id',
     ID_TYPE_USER_BY_EMAIL: 'email_address',
     ID_TYPE_USER_BY_ID: 'id',
 }
 
 
+class AnonymousGSConnection(GSConnection):
+  """The GSConnection class constructor in

[rmistry, 2014/07/16 13:55:01]

The Nittiest of Nits:
The style guide says to have a one line summary followed by a blank line and
then more description.

"""GSConnection class that allows anonymous connections.

Your current description.
"""

[epoger, 2014/07/16 14:02:02]

I'm not the biggest fan of that directive in the style guide.  ("You have to
indent this much, and you only get 80 chars per line, and you have to fit the
description in one line.")  I am tempted to describe each function only as "See.
Run. Love."

But thank you for suggesting a reasonable one-liner to put at the top.  Done.

+  https://github.com/boto/boto/blob/develop/boto/gs/connection.py doesn't allow
+  for anonymous connections (connections without credentials), so we have to
+  override it.
+  """
+  def __init__(self):
+    super(GSConnection, self).__init__(
+        # This is the important bit we need to add...
+        anon=True,
+        # ...and these are just copied in from GSConnection.__init__()
+        bucket_class=Bucket,
+        calling_format=SubdomainCallingFormat(),
+        host=GSConnection.DefaultHost,
+        provider='google')
+
+
 class GSUtils(object):
   """Utilities for accessing Google Cloud Storage, using the boto library."""
 
-  def __init__(self, boto_file_path=os.path.join('~','.boto')):
+  def __init__(self, boto_file_path=None):
     """Constructor.
 
     Params:
       boto_file_path: full path (local-OS-style) on local disk where .boto
-          credentials file can be found.  An exception is thrown if this file
-          is missing.
-          TODO(epoger): Change missing-file behavior: allow the caller to
-          operate on public files in Google Storage.
+          credentials file can be found.  If None, then the GSUtils object
+          created will be able to access only public files in Google Storage.
+
+    Raises an exception if no file is found at boto_file_path, or if the file
+    found there is malformed.
     """
-    boto_file_path = os.path.expanduser(boto_file_path)
-    print 'Reading boto file from %s' % boto_file_path
-    boto_dict = _config_file_as_dict(filepath=boto_file_path)
-    self._gs_access_key_id = boto_dict['gs_access_key_id']
-    self._gs_secret_access_key = boto_dict['gs_secret_access_key']
+    self._gs_access_key_id = None
+    self._gs_secret_access_key = None
+    if boto_file_path:
+      print 'Reading boto file from %s' % boto_file_path
+      boto_dict = _config_file_as_dict(filepath=boto_file_path)
+      self._gs_access_key_id = boto_dict['gs_access_key_id']
+      self._gs_secret_access_key = boto_dict['gs_secret_access_key']
 
   def delete_file(self, bucket, path):
     """Delete a single file within a GS bucket.
 
     TODO(epoger): what if bucket or path does not exist?  Should probably raise
     an exception.  Implement, and add a test to exercise this.
 
     Params:
       bucket: GS bucket to delete a file from
       path: full path (Posix-style) of the file within the bucket to delete
@@ skipping 153 matching lines @@
     for item in lister:
       t = type(item)
       if t is Key:
         files.append(item.key[prefix_length:])
       elif t is Prefix:
         dirs.append(item.name[prefix_length:-1])
     return (dirs, files)
 
   def _create_connection(self):
     """Returns a GSConnection object we can use to access Google Storage."""
-    return GSConnection(
-        gs_access_key_id=self._gs_access_key_id,
-        gs_secret_access_key=self._gs_secret_access_key)
-
+    if self._gs_access_key_id:
+      return GSConnection(
+          gs_access_key_id=self._gs_access_key_id,
+          gs_secret_access_key=self._gs_secret_access_key)
+    else:
+      return AnonymousGSConnection()
 
 def _config_file_as_dict(filepath):
   """Reads a boto-style config file into a dict.
 
   Parses all lines from the file of this form: key = value
   TODO(epoger): Create unittest.
 
   Params:
     filepath: path to config file on local disk
 
@@ skipping 19 matching lines @@
   Args:
     path: full path of directory to create
   """
   try:
     os.makedirs(path)
   except OSError as e:
     if e.errno != errno.EEXIST:
       raise
 
 
-def _run_self_test():
+def _test_public_read():
+  """Make sure we can read from public files without .boto file credentials."""
+  gs = GSUtils()
+  gs.list_bucket_contents(bucket='chromium-skia-gm-summaries', subdir=None)
+
+
+def _test_authenticated_round_trip():
+  try:
+    gs = GSUtils(boto_file_path=os.path.expanduser(os.path.join('~','.boto')))
+  except:
+    print """
+Failed to instantiate GSUtils object with default .boto file path.
+Do you have a ~/.boto file that provides the credentials needed to read
+and write gs://chromium-skia-gm ?
+"""
+    raise
+
   bucket = 'chromium-skia-gm'
   remote_dir = 'gs_utils_test/%d' % random.randint(0, sys.maxint)
   subdir = 'subdir'
   filenames_to_upload = ['file1', 'file2']
-  gs = GSUtils()
 
   # Upload test files to Google Storage.
   local_src_dir = tempfile.mkdtemp()
   os.mkdir(os.path.join(local_src_dir, subdir))
   try:
     for filename in filenames_to_upload:
       with open(os.path.join(local_src_dir, subdir, filename), 'w') as f:
         f.write('contents of %s\n' % filename)
       gs.upload_file(source_path=os.path.join(local_src_dir, subdir, filename),
                      dest_bucket=bucket,
@@ skipping 66 matching lines @@
     gs.delete_file(bucket=bucket,
                    path=posixpath.join(remote_dir, subdir, filename))
 
   # Confirm that we deleted all the files we uploaded to Google Storage.
   (dirs, files) = gs.list_bucket_contents(
       bucket=bucket, subdir=posixpath.join(remote_dir, subdir))
   assert dirs == [], '%s == []' % dirs
   assert files == [], '%s == []' % files
 
 
-# TODO(epoger): How should we exercise this self-test?
-# I avoided using the standard unittest framework, because these Google Storage
-# operations are expensive and require .boto permissions.
-#
-# How can we automatically test this code without wasting too many resources
-# or needing .boto permissions?
+# TODO(epoger): How should we exercise these self-tests?
+# See http://skbug.com/2751
 if __name__ == '__main__':
-  _run_self_test()
+  _test_public_read()
+  _test_authenticated_round_trip()
+  # TODO(epoger): Add _test_unauthenticated_access() to make sure we raise
+  # an exception when we try to access without needed credentials.