allow upload_dir_contents() to write into a bucket's root dir

py/utils/gs_utils.py

 #!/usr/bin/python
 
 # pylint: disable=C0301
 """
 Copyright 2014 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Utilities for accessing Google Cloud Storage, using the boto library (wrapper
@@ skipping 179 matching lines @@
 
   def upload_dir_contents(self, source_dir, dest_bucket, dest_dir,
                           predefined_acl=None, fine_grained_acl_list=None):
     """Recursively upload contents of a local directory to Google Storage.
 
     params:
       source_dir: full path (local-OS-style) on local disk of directory to copy
           contents of
       dest_bucket: GCS bucket to copy the files into
       dest_dir: full path (Posix-style) within that bucket; write the files into
-          this directory
+          this directory.  If None, write into the root directory of the bucket.
       predefined_acl: which predefined ACL to apply to the files on Google
           Storage; must be one of the PredefinedACL values defined above.
           If None, inherits dest_bucket's default object ACL.
           TODO(epoger): add unittests for this param, although it seems to work
           in my manual testing
       fine_grained_acl_list: list of (id_type, id_value, permission) tuples
           to apply to every file uploaded (on top of the predefined_acl),
           or None if predefined_acl is sufficient
 
     The copy operates as a "merge with overwrite": any files in source_dir will
     be "overlaid" on top of the existing content in dest_dir.  Existing files
     with the same names will be overwritten.
 
     TODO(epoger): Upload multiple files simultaneously to reduce latency.
 
     TODO(epoger): Add a "noclobber" mode that will not upload any files would
     overwrite existing files in Google Storage.
 
     TODO(epoger): Consider adding a do_compress parameter that would compress
     the file using gzip before upload, and add a "Content-Encoding:gzip" header
     so that HTTP downloads of the file would be unzipped automatically.
     See https://developers.google.com/storage/docs/gsutil/addlhelp/
         WorkingWithObjectMetadata#content-encoding
     """
     b = self._connect_to_bucket(bucket_name=dest_bucket)
     for filename in sorted(os.listdir(source_dir)):
       local_path = os.path.join(source_dir, filename)
+      if dest_dir:
+        remote_path = posixpath.join(dest_dir, filename)
+      else:
+        remote_path = filename
+
       if os.path.isdir(local_path):
         self.upload_dir_contents(  # recurse
             source_dir=local_path, dest_bucket=dest_bucket,
-            dest_dir=posixpath.join(dest_dir, filename),
+            dest_dir=remote_path,
             predefined_acl=predefined_acl,
             fine_grained_acl_list=fine_grained_acl_list)
       else:
         item = Key(b)
-        dest_path = posixpath.join(dest_dir, filename)
-        item.key = dest_path
+        item.key = remote_path
         try:
           item.set_contents_from_filename(
               filename=local_path, policy=predefined_acl)
         except BotoServerError, e:
           e.body = (repr(e.body) +
                     ' while uploading local_path=%s to bucket=%s, path=%s' % (
-                        local_path, dest_bucket, dest_path))
+                        local_path, dest_bucket, remote_path))
           raise
         # TODO(epoger): This may be inefficient, because it calls
         # _connect_to_bucket() for every file.  Depending on how expensive that
         # call is, we may want to optimize this.
         for (id_type, id_value, permission) in fine_grained_acl_list or []:
           self.set_acl(
-              bucket=dest_bucket, path=dest_path,
+              bucket=dest_bucket, path=remote_path,
               id_type=id_type, id_value=id_value, permission=permission)
 
   def download_file(self, source_bucket, source_path, dest_path,
                     create_subdirs_if_needed=False):
     """Downloads a single file from Google Cloud Storage to local disk.
 
     Args:
       source_bucket: GCS bucket to download the file from
       source_path: full path (Posix-style) within that bucket
       dest_path: full path (local-OS-style) on local disk to copy the file to
@@ skipping 227 matching lines @@
   exist yet.
 
   Args:
     path: full path of directory to create
   """
   try:
     os.makedirs(path)
   except OSError as e:
     if e.errno != errno.EEXIST:
       raise