Add does_storage_object_exist to gs_utils

py/utils/gs_utils.py

 #!/usr/bin/python
 
 # pylint: disable=C0301
 """
 Copyright 2014 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Utilities for accessing Google Cloud Storage, using the boto library (wrapper
@@ skipping 34 matching lines @@
 from boto.s3.connection import SubdomainCallingFormat
 from boto.s3.prefix import Prefix
 
 # How many files to upload at once, by default.
 # TODO(epoger): Is there a way to compute this intelligently?  To some extent
 # it is a function of how many cores are on the machine, and how many other
 # processes it is running; but it's probably more a function of how much time
 # each core sits idle waiting for network I/O to complete.
 DEFAULT_UPLOAD_THREADS = 10
 
-_GS_PREFIX = 'gs://'
+GS_PREFIX = 'gs://'
 
 
 class AnonymousGSConnection(GSConnection):
   """GSConnection class that allows anonymous connections.
 
   The GSConnection class constructor in
   https://github.com/boto/boto/blob/develop/boto/gs/connection.py doesn't allow
   for anonymous connections (connections without credentials), so we have to
   override it.
   """
@@ skipping 59 matching lines @@
     IF_NEW = 2      # if there is an existing file with the same name,
                     # leave it alone
     IF_MODIFIED = 3 # if there is an existing file with the same name and
                     # contents, leave it alone
 
   def __init__(self, boto_file_path=None):
     """Constructor.
 
     Params:
       boto_file_path: full path (local-OS-style) on local disk where .boto
-          credentials file can be found.  If None, then the GSUtils object
-          created will be able to access only public files in Google Storage.
+          credentials file can be found.  If None, fall back on the
+          AWS_CREDENTIAL_FILE environment variable, then look in a set of
+          common paths for the .boto file.  If no .boto file is found, then the
+          GSUtils object created will be able to access only public files in
+          Google Storage.
 
     Raises an exception if no file is found at boto_file_path, or if the file
     found there is malformed.
     """
     self._gs_access_key_id = None
     self._gs_secret_access_key = None
+    if not boto_file_path:
+      if os.environ.get('AWS_CREDENTIAL_FILE'):
+        boto_file_path = os.path.expanduser(os.environ['AWS_CREDENTIAL_FILE'])
+    if not boto_file_path:

[rmistry, 2014/10/16 18:09:58]

Remove this if since you already have it above.

[borenet, 2014/10/16 18:12:51]

This is only intended to run if AWS_CREDENTIAL_FILE is not defined.  Otherwise
we'd be falling back unnecessarily on ~/.boto.

+      for path in (os.path.join(os.path.expanduser('~'), '.boto'),):
+        if os.path.isfile(path):
+          boto_file_path = path
+          break
+
     if boto_file_path:
       print ('Reading boto file from %s' % boto_file_path)
       boto_dict = _config_file_as_dict(filepath=boto_file_path)
       self._gs_access_key_id = boto_dict['gs_access_key_id']
       self._gs_secret_access_key = boto_dict['gs_secret_access_key']
+    else:
+      print >> sys.stderr, 'Warning: no .boto file found.'
+
     # Which field we get/set in ACL entries, depending on IdType.
     self._field_by_id_type = {
         self.IdType.GROUP_BY_DOMAIN: 'domain',
         self.IdType.GROUP_BY_EMAIL:  'email_address',
         self.IdType.GROUP_BY_ID:     'id',
         self.IdType.USER_BY_EMAIL:   'email_address',
         self.IdType.USER_BY_ID:      'id',
     }
 
   def delete_file(self, bucket, path):
@@ skipping 417 matching lines @@
     dirs = []
     files = []
     for item in items:
       t = type(item)
       if t is Key:
         files.append(item.name[prefix_length:])
       elif t is Prefix:
         dirs.append(item.name[prefix_length:-1])
     return (dirs, files)
 
+  def does_storage_object_exist(self, bucket, object_name):
+    """Determines whether an object exists in Google Storage.
+
+    Returns True if it exists else returns False.
+    """
+    b = self._connect_to_bucket(bucket=bucket)
+    if object_name in b:
+      return True
+    dirs, files = self.list_bucket_contents(bucket, object_name)
+    return bool(dirs or files)
+
   @staticmethod
   def is_gs_url(url):
     """Returns True if url is a legal Google Storage URL ("gs://bucket/file").
     """
     try:
-      if url.lower().startswith(_GS_PREFIX) and len(url) > len(_GS_PREFIX):
-        return url[len(_GS_PREFIX)].isalnum()
+      if url.lower().startswith(GS_PREFIX) and len(url) > len(GS_PREFIX):
+        return url[len(GS_PREFIX)].isalnum()
       else:
         return False
     except AttributeError:
       return False
 
   @staticmethod
   def split_gs_url(url):
     """Returns (bucket, filepath) corresponding to a legal Google Storage URL.
 
     Raises AttributeError if the input URL is not a legal Google Storage URL.
     """
     if not GSUtils.is_gs_url(url):
       raise AttributeError('"%s" is not a legal Google Storage URL' % url)
-    prefix_removed = url[len(_GS_PREFIX):]
+    prefix_removed = url[len(GS_PREFIX):]
     pathsep_index = prefix_removed.find('/')
     if pathsep_index < 0:
       return (prefix_removed, '')
     else:
       return (prefix_removed[:pathsep_index],
               prefix_removed[pathsep_index+1:].strip('/'))
 
   def _connect_to_bucket(self, bucket):
     """Returns a Bucket object we can use to access a particular bucket in GS.
 
@@ skipping 59 matching lines @@
 
 def _get_local_md5(path):
   """Returns the MD5 hash of a file on local disk."""
   hasher = hashlib.md5()
   with open(path, 'rb') as f:
     while True:
       data = f.read(64*1024)
       if not data:
         return hasher.hexdigest()
       hasher.update(data)