fix fine_grained_acl_list in upload_dir_contents(), and add unittest

py/utils/gs_utils.py

 #!/usr/bin/python
 
 # pylint: disable=C0301
 """
 Copyright 2014 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Utilities for accessing Google Cloud Storage, using the boto library (wrapper
@@ skipping 184 matching lines @@
       dest_dir: full path (Posix-style) within that bucket; write the files into
           this directory
       predefined_acl: which predefined ACL to apply to the files on Google
           Storage; must be one of the PREDEFINED_ACL_* constants defined above.
           If None, inherits dest_bucket's default object ACL.
           TODO(epoger): add unittests for this param, although it seems to work
           in my manual testing
       fine_grained_acl_list: list of (id_type, id_value, permission) tuples
           to apply to every file uploaded (on top of the predefined_acl),
           or None if predefined_acl is sufficient
-          TODO(epoger): add unittests for this param, although it seems to work
-          in my manual testing
 
     The copy operates as a "merge with overwrite": any files in source_dir will
     be "overlaid" on top of the existing content in dest_dir.  Existing files
     with the same names will be overwritten.
 
     TODO(epoger): Upload multiple files simultaneously to reduce latency.
 
     TODO(epoger): Add a "noclobber" mode that will not upload any files would
     overwrite existing files in Google Storage.
 
     TODO(epoger): Consider adding a do_compress parameter that would compress
     the file using gzip before upload, and add a "Content-Encoding:gzip" header
     so that HTTP downloads of the file would be unzipped automatically.
     See https://developers.google.com/storage/docs/gsutil/addlhelp/
         WorkingWithObjectMetadata#content-encoding
     """
     b = self._connect_to_bucket(bucket_name=dest_bucket)
     for filename in sorted(os.listdir(source_dir)):
       local_path = os.path.join(source_dir, filename)
       if os.path.isdir(local_path):
         self.upload_dir_contents(  # recurse
             source_dir=local_path, dest_bucket=dest_bucket,
             dest_dir=posixpath.join(dest_dir, filename),
-            predefined_acl=predefined_acl)
+            predefined_acl=predefined_acl,
+            fine_grained_acl_list=fine_grained_acl_list)

[epoger, 2014/07/18 20:41:26]

whoops

       else:
         item = Key(b)
-        item.key = posixpath.join(dest_dir, filename)
+        dest_path = posixpath.join(dest_dir, filename)
+        item.key = dest_path
         try:
           item.set_contents_from_filename(
               filename=local_path, policy=predefined_acl)
         except BotoServerError, e:
           e.body = (repr(e.body) +
                     ' while uploading local_path=%s to bucket=%s, path=%s' % (
-                        local_path, dest_bucket, item.key))
+                        local_path, dest_bucket, dest_path))
           raise
         # TODO(epoger): This may be inefficient, because it calls
         # _connect_to_bucket() for every file.  Depending on how expensive that
         # call is, we may want to optimize this.
         for (id_type, id_value, permission) in fine_grained_acl_list or []:
           self.set_acl(
-              bucket=dest_bucket, path=item.key,
+              bucket=dest_bucket, path=dest_path,
               id_type=id_type, id_value=id_value, permission=permission)
 
   def download_file(self, source_bucket, source_path, dest_path,
                     create_subdirs_if_needed=False):
     """Downloads a single file from Google Cloud Storage to local disk.
 
     Args:
       source_bucket: GCS bucket to download the file from
       source_path: full path (Posix-style) within that bucket
       dest_path: full path (local-OS-style) on local disk to copy the file to
@@ skipping 362 matching lines @@
 Do you have a ~/.boto file that provides the credentials needed to read
 and write gs://chromium-skia-gm ?
 """
     raise
 
   bucket = 'chromium-skia-gm'
   remote_dir = 'gs_utils_test/%d' % random.randint(0, sys.maxint)
   subdir = 'subdir'
   filenames = ['file1', 'file2']
 
-  # Create directory tree on local disk, and upload it.
+  # Create directory tree on local disk and upload it.
+  id_type = ID_TYPE_GROUP_BY_DOMAIN
+  id_value = 'chromium.org'
+  set_permission = PERMISSION_READ
   local_src_dir = tempfile.mkdtemp()
   os.mkdir(os.path.join(local_src_dir, subdir))
   try:
     for filename in filenames:
       with open(os.path.join(local_src_dir, subdir, filename), 'w') as f:
         f.write('contents of %s\n' % filename)
-      gs.upload_dir_contents(source_dir=local_src_dir, dest_bucket=bucket,
-                             dest_dir=remote_dir)
+    gs.upload_dir_contents(

[epoger, 2014/07/18 20:41:26]

This was indented wrong before (and calling upload_dir_contents() multiple
times)

+        source_dir=local_src_dir, dest_bucket=bucket, dest_dir=remote_dir,
+        predefined_acl=PREDEFINED_ACL_PRIVATE,
+        fine_grained_acl_list=[(id_type, id_value, set_permission)])
   finally:
     shutil.rmtree(local_src_dir)
 
   # Validate the list of the files we uploaded to Google Storage.
   (dirs, files) = gs.list_bucket_contents(
       bucket=bucket, subdir=remote_dir)
   assert dirs == [subdir], '%s == [%s]' % (dirs, subdir)
   assert files == [], '%s == []' % files
   (dirs, files) = gs.list_bucket_contents(
       bucket=bucket, subdir=posixpath.join(remote_dir, subdir))
   assert dirs == [], '%s == []' % dirs
   assert files == filenames, '%s == %s' % (files, filenames)
 
+  # Check the fine-grained ACLs we set in Google Storage.
+  for filename in filenames:
+    got_permission = gs.get_acl(
+        bucket=bucket, path=posixpath.join(remote_dir, subdir, filename),
+        id_type=id_type, id_value=id_value)
+    assert got_permission == set_permission, '%s == %s' % (
+        got_permission, set_permission)
+
   # Download the directory tree we just uploaded, make sure its contents
   # are what we expect, and then delete the tree in Google Storage.
   local_dest_dir = tempfile.mkdtemp()
   try:
     gs.download_dir_contents(source_bucket=bucket, source_dir=remote_dir,
                              dest_dir=local_dest_dir)
     for filename in filenames:
       with open(os.path.join(local_dest_dir, subdir, filename)) as f:
         file_contents = f.read()
       assert file_contents == 'contents of %s\n' % filename, (
           '%s == "contents of %s\n"' % (file_contents, filename))
   finally:
     shutil.rmtree(local_dest_dir)
     for filename in filenames:
       gs.delete_file(bucket=bucket,
                      path=posixpath.join(remote_dir, subdir, filename))
 
 
 # TODO(epoger): How should we exercise these self-tests?
 # See http://skbug.com/2751
 if __name__ == '__main__':
   _test_public_read()
   _test_authenticated_round_trip()
   _test_dir_upload_and_download()
   # TODO(epoger): Add _test_unauthenticated_access() to make sure we raise
   # an exception when we try to access without needed credentials.