add set_acl() and get_acl() to gs_utils.py

py/utils/gs_utils.py

 #!/usr/bin/python
 
 # pylint: disable=C0301
 """
 Copyright 2014 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
-Utilities for accessing Google Cloud Storage, using the boto library.
+Utilities for accessing Google Cloud Storage, using the boto library (wrapper
+for the XML API).
 
-See http://googlecloudstorage.blogspot.com/2012/09/google-cloud-storage-tutorial-using-boto.html
-for implementation tips.
+API/library references:
+- https://developers.google.com/storage/docs/reference-guide
+- http://googlecloudstorage.blogspot.com/2012/09/google-cloud-storage-tutorial-using-boto.html
 """
 # pylint: enable=C0301
 
 # System-level imports
 import errno
 import os
 import posixpath
 import random
 import re
 import shutil
 import sys
 import tempfile
 
 # Imports from third-party code
 TRUNK_DIRECTORY = os.path.abspath(os.path.join(
     os.path.dirname(__file__), os.pardir, os.pardir))
 for import_subdir in ['boto']:
   import_dirpath = os.path.join(
       TRUNK_DIRECTORY, 'third_party', 'externals', import_subdir)
   if import_dirpath not in sys.path:
     # We need to insert at the beginning of the path, to make sure that our
     # imported versions are favored over others that might be in the path.
     sys.path.insert(0, import_dirpath)
+from boto.gs import acl
 from boto.gs.connection import GSConnection
 from boto.gs.key import Key
 from boto.s3.bucketlistresultset import BucketListResultSet
 from boto.s3.prefix import Prefix
 
+# SupportedPermissions as listed in
+# https://github.com/boto/boto/blob/develop/boto/gs/acl.py
+PERMISSION_OWNER = 'FULL_CONTROL'
+PERMISSION_READ  = 'READ'
+PERMISSION_WRITE = 'WRITE'
+
+# Types of identifiers we can use to set ACLs.
+ID_TYPE_GROUP_BY_DOMAIN = acl.GROUP_BY_DOMAIN
+ID_TYPE_GROUP_BY_EMAIL = acl.GROUP_BY_EMAIL
+ID_TYPE_GROUP_BY_ID = acl.GROUP_BY_ID
+ID_TYPE_USER_BY_EMAIL = acl.USER_BY_EMAIL
+ID_TYPE_USER_BY_ID = acl.USER_BY_ID
+
+# Which field we get/set in ACL entries, depending on ID_TYPE.
+FIELD_BY_ID_TYPE = {
+    ID_TYPE_GROUP_BY_DOMAIN: 'domain',
+    ID_TYPE_GROUP_BY_EMAIL: 'email_address',
+    ID_TYPE_GROUP_BY_ID: 'id',
+    ID_TYPE_USER_BY_EMAIL: 'email_address',
+    ID_TYPE_USER_BY_ID: 'id',
+}
+
 
 class GSUtils(object):
   """Utilities for accessing Google Cloud Storage, using the boto library."""
 
   def __init__(self, boto_file_path=os.path.join('~','.boto')):
     """Constructor.
 
     Params:
       boto_file_path: full path (local-OS-style) on local disk where .boto
           credentials file can be found.  An exception is thrown if this file
@@ skipping 54 matching lines @@
     """
     conn = self._create_connection()
     b = conn.get_bucket(bucket_name=source_bucket)
     item = Key(b)
     item.key = source_path
     if create_subdirs_if_needed:
       _makedirs_if_needed(os.path.dirname(dest_path))
     with open(dest_path, 'w') as f:
       item.get_contents_to_file(fp=f)
 
+  def add_acl(self, bucket, path, id_type, id_value, permission):
+    """Add access permissions on a single file in Google Storage.
+
+    After this call, the set of users with access rights will always be >=
+    the set of users with access rights before the call, because the permissions
+    are additive.
+    (E.g., if you add READ permission for a group, but a member of that group
+    already has WRITE permission, that member will still have WRITE permission.)
+    TODO(epoger): Do we know that for sure?  I *think* that's how it works...
+
+    If there is already a permission set for this id_type/id_value combination,
+    this call will overwrite it.
+
+    Params:
+      bucket: GS bucket
+      path: full path (Posix-style) to the file within that bucket
+      id_type: must be one of the ID_TYPE_* constants defined above
+      id_value: add permission for users whose id_type field contains this value
+      permission: permission to add for users matching id_type/id_value;
+          must be one of the PERMISSION_* constants defined above
+    """
+    field = FIELD_BY_ID_TYPE[id_type]
+    conn = self._create_connection()
+    b = conn.get_bucket(bucket_name=bucket)
+    acls = b.get_acl(key_name=path)
+
+    # Remove any existing entries that refer to the same id_type/id_value,
+    # because the API will fail if we try to set more than one.
+    matching_entries = [entry for entry in acls.entries.entry_list
+                        if (entry.scope.type == id_type) and
+                        (getattr(entry.scope, field) == id_value)]
+    if matching_entries:
+      for entry in matching_entries:

[rmistry, 2014/07/15 11:26:10]

Is it possible to ever get a list of more than one entry here? i.e. is this
true: there can be only one entry with the same id_type and id_value.

I see you are asserting for this in get_acl but not in add_acl and delete_acl
and I was wondering why, if that block of code can be identical in all three
methods then we could extract it into a helper method.

[epoger, 2014/07/15 13:20:18]

AFAICT we should always get either 0 or 1 matching_entries.  I should probably
go ahead and add the assert here; when that assert fails, that will tell me I'm
wrong about that assumption. :-)

+        acls.entries.entry_list.remove(entry)
+
+    # Add a new entry to the ACLs.
+    args = {'type': id_type, 'permission': permission}
+    args[field] = id_value
+    new_entry = acl.Entry(**args)
+    acls.entries.entry_list.append(new_entry)
+    b.set_acl(acl_or_str=acls, key_name=path)
+
+  def delete_acl(self, bucket, path, id_type, id_value):
+    """Delete certain access permissions on a single file in Google Storage.
+
+    Various users who match this id_type/id_value pair may still have access
+    rights to this file after this call, if they have been granted those rights
+    based on *other* id_types (e.g., perhaps they still have individual user
+    access rights, even if their group access rights are removed).
+
+    If no permissions have been added for this id_type/id_value, this will
+    return uneventfully (there will be no exception or other indication of
+    failure).
+
+    Params:
+      bucket: GS bucket
+      path: full path (Posix-style) to the file within that bucket
+      id_type: must be one of the ID_TYPE_* constants defined above
+      id_value: delete permissions for users whose id_type field contains this
+          value
+    """
+    field = FIELD_BY_ID_TYPE[id_type]
+    conn = self._create_connection()
+    b = conn.get_bucket(bucket_name=bucket)
+    acls = b.get_acl(key_name=path)
+    matching_entries = [entry for entry in acls.entries.entry_list
+                        if (entry.scope.type == id_type) and
+                        (getattr(entry.scope, field) == id_value)]
+    if matching_entries:
+      for entry in matching_entries:
+        acls.entries.entry_list.remove(entry)
+      b.set_acl(acl_or_str=acls, key_name=path)
+
+  def get_acl(self, bucket, path, id_type, id_value):
+    """Retrieve partial access permissions on a single file in Google Storage.
+
+    Various users who match this id_type/id_value pair may have access rights
+    other than that returned by this call, if they have been granted those
+    rights based on *other* id_types (e.g., perhaps they have group access
+    rights, beyond their individual access rights).
+
+    Params:
+      bucket: GS bucket
+      path: full path (Posix-style) to the file within that bucket
+      id_type: must be one of the ID_TYPE_* constants defined above
+      id_value: delete permissions for users whose id_type field contains this

[epoger, 2014/07/15 13:34:26]

delete -> get

+          value
+

[epoger, 2014/07/15 13:34:26]

Add an example

+    Returns: the PERMISSION_* constant which has been set for users matching
+        this id_type/id_value, on this file; or None if no such permissions have
+        been set.

[rmistry, 2014/07/15 11:26:10]

[Optional] How about creating a PERMISSION_NONE or PERMISSION_MISSING that is =
None.

[epoger, 2014/07/15 13:20:18]

I can do that, *if* you think that would also make sense to be passed into
add_acl() as a value of the permission param.  I guess that would replace
delete_acl()...

+    """
+    field = FIELD_BY_ID_TYPE[id_type]
+    conn = self._create_connection()
+    b = conn.get_bucket(bucket_name=bucket)
+    acls = b.get_acl(key_name=path)
+    matching_entries = [entry for entry in acls.entries.entry_list
+                        if (entry.scope.type == id_type) and
+                        (getattr(entry.scope, field) == id_value)]
+    if matching_entries:
+      assert len(matching_entries) == 1, '%d == 1' % len(matching_entries)
+      return matching_entries[0].permission
+    else:
+      return None
+
   def list_bucket_contents(self, bucket, subdir=None):
     """Returns files in the Google Storage bucket as a (dirs, files) tuple.
 
     Args:
       bucket: name of the Google Storage bucket
       subdir: directory within the bucket to list, or None for root directory
     """
     # The GS command relies on the prefix (if any) ending with a slash.
     prefix = subdir or ''
     if prefix and not prefix.endswith('/'):
@@ skipping 81 matching lines @@
   # Get a list of the files we uploaded to Google Storage.
   (dirs, files) = gs.list_bucket_contents(
       bucket=bucket, subdir=remote_dir)
   assert dirs == [subdir], '%s == [%s]' % (dirs, subdir)
   assert files == [], '%s == []' % files
   (dirs, files) = gs.list_bucket_contents(
       bucket=bucket, subdir=posixpath.join(remote_dir, subdir))
   assert dirs == [], '%s == []' % dirs
   assert files == filenames_to_upload, '%s == %s' % (files, filenames_to_upload)
 
+  # Manipulate ACLs on one of those files, and verify them.
+  # TODO(epoger): Test id_types other than ID_TYPE_GROUP_BY_DOMAIN ?
+  # TODO(epoger): Test setting multiple ACLs on the same file?
+  id_type = ID_TYPE_GROUP_BY_DOMAIN
+  id_value = 'google.com'
+  fullpath = posixpath.join(remote_dir, subdir, filenames_to_upload[0])
+  # Make sure ACL is empty to start with ...
+  gs.delete_acl(bucket=bucket, path=fullpath,
+                id_type=id_type, id_value=id_value)
+  permission = gs.get_acl(bucket=bucket, path=fullpath,
+                          id_type=id_type, id_value=id_value)
+  assert permission == None, '%s == None' % permission
+  # ... set it to OWNER ...
+  gs.add_acl(bucket=bucket, path=fullpath,
+             id_type=id_type, id_value=id_value, permission=PERMISSION_OWNER)
+  permission = gs.get_acl(bucket=bucket, path=fullpath,
+                          id_type=id_type, id_value=id_value)
+  assert permission == PERMISSION_OWNER, '%s == %s' % (
+      permission, PERMISSION_OWNER)
+  # ... now set it to READ ...
+  gs.add_acl(bucket=bucket, path=fullpath,
+             id_type=id_type, id_value=id_value, permission=PERMISSION_READ)
+  permission = gs.get_acl(bucket=bucket, path=fullpath,
+                          id_type=id_type, id_value=id_value)
+  assert permission == PERMISSION_READ, '%s == %s' % (
+      permission, PERMISSION_READ)
+  # ... and clear it again to finish.
+  gs.delete_acl(bucket=bucket, path=fullpath,
+                id_type=id_type, id_value=id_value)
+  permission = gs.get_acl(bucket=bucket, path=fullpath,
+                          id_type=id_type, id_value=id_value)
+  assert permission == None, '%s == None' % permission
+
   # Download the files we uploaded to Google Storage, and validate contents.
   local_dest_dir = tempfile.mkdtemp()
   try:
     for filename in filenames_to_upload:
       gs.download_file(source_bucket=bucket,
                        source_path=posixpath.join(remote_dir, subdir, filename),
                        dest_path=os.path.join(local_dest_dir, subdir, filename),
                        create_subdirs_if_needed=True)
       with open(os.path.join(local_dest_dir, subdir, filename)) as f:
         file_contents = f.read()
@@ skipping 15 matching lines @@
 
 
 # TODO(epoger): How should we exercise this self-test?
 # I avoided using the standard unittest framework, because these Google Storage
 # operations are expensive and require .boto permissions.
 #
 # How can we automatically test this code without wasting too many resources
 # or needing .boto permissions?
 if __name__ == '__main__':
   _run_self_test()