Index: py/utils/gs_utils.py |
diff --git a/py/utils/gs_utils.py b/py/utils/gs_utils.py |
index 46c9bf17afe2aa9bc654135f02972e2f57a35320..8ba273487a1b7d816fa07a3415b423140e21a72b 100755 |
--- a/py/utils/gs_utils.py |
+++ b/py/utils/gs_utils.py |
@@ -7,10 +7,12 @@ Copyright 2014 Google Inc. |
Use of this source code is governed by a BSD-style license that can be |
found in the LICENSE file. |
-Utilities for accessing Google Cloud Storage, using the boto library. |
+Utilities for accessing Google Cloud Storage, using the boto library (wrapper |
+for the XML API). |
-See http://googlecloudstorage.blogspot.com/2012/09/google-cloud-storage-tutorial-using-boto.html |
-for implementation tips. |
+API/library references: |
+- https://developers.google.com/storage/docs/reference-guide |
+- http://googlecloudstorage.blogspot.com/2012/09/google-cloud-storage-tutorial-using-boto.html |
""" |
# pylint: enable=C0301 |
@@ -34,11 +36,34 @@ for import_subdir in ['boto']: |
# We need to insert at the beginning of the path, to make sure that our |
# imported versions are favored over others that might be in the path. |
sys.path.insert(0, import_dirpath) |
+from boto.gs import acl |
from boto.gs.connection import GSConnection |
from boto.gs.key import Key |
from boto.s3.bucketlistresultset import BucketListResultSet |
from boto.s3.prefix import Prefix |
+# SupportedPermissions as listed in |
+# https://github.com/boto/boto/blob/develop/boto/gs/acl.py |
+PERMISSION_OWNER = 'FULL_CONTROL' |
+PERMISSION_READ = 'READ' |
+PERMISSION_WRITE = 'WRITE' |
+ |
+# Types of identifiers we can use to set ACLs. |
+ID_TYPE_GROUP_BY_DOMAIN = acl.GROUP_BY_DOMAIN |
+ID_TYPE_GROUP_BY_EMAIL = acl.GROUP_BY_EMAIL |
+ID_TYPE_GROUP_BY_ID = acl.GROUP_BY_ID |
+ID_TYPE_USER_BY_EMAIL = acl.USER_BY_EMAIL |
+ID_TYPE_USER_BY_ID = acl.USER_BY_ID |
+ |
+# Which field we get/set in ACL entries, depending on ID_TYPE. |
+FIELD_BY_ID_TYPE = { |
+ ID_TYPE_GROUP_BY_DOMAIN: 'domain', |
+ ID_TYPE_GROUP_BY_EMAIL: 'email_address', |
+ ID_TYPE_GROUP_BY_ID: 'id', |
+ ID_TYPE_USER_BY_EMAIL: 'email_address', |
+ ID_TYPE_USER_BY_ID: 'id', |
+} |
+ |
class GSUtils(object): |
"""Utilities for accessing Google Cloud Storage, using the boto library.""" |
@@ -113,6 +138,111 @@ class GSUtils(object): |
with open(dest_path, 'w') as f: |
item.get_contents_to_file(fp=f) |
+ def add_acl(self, bucket, path, id_type, id_value, permission): |
+ """Add access permissions on a single file in Google Storage. |
+ |
+ After this call, the set of users with access rights will always be >= |
+ the set of users with access rights before the call, because the permissions |
+ are additive. |
+ (E.g., if you add READ permission for a group, but a member of that group |
+ already has WRITE permission, that member will still have WRITE permission.) |
+ TODO(epoger): Do we know that for sure? I *think* that's how it works... |
+ |
+ If there is already a permission set for this id_type/id_value combination, |
+ this call will overwrite it. |
+ |
+ Params: |
+ bucket: GS bucket |
+ path: full path (Posix-style) to the file within that bucket |
+ id_type: must be one of the ID_TYPE_* constants defined above |
+ id_value: add permission for users whose id_type field contains this value |
+ permission: permission to add for users matching id_type/id_value; |
+ must be one of the PERMISSION_* constants defined above |
+ """ |
+ field = FIELD_BY_ID_TYPE[id_type] |
+ conn = self._create_connection() |
+ b = conn.get_bucket(bucket_name=bucket) |
+ acls = b.get_acl(key_name=path) |
+ |
+ # Remove any existing entries that refer to the same id_type/id_value, |
+ # because the API will fail if we try to set more than one. |
+ matching_entries = [entry for entry in acls.entries.entry_list |
+ if (entry.scope.type == id_type) and |
+ (getattr(entry.scope, field) == id_value)] |
+ if matching_entries: |
+ for entry in matching_entries: |
rmistry
2014/07/15 11:26:10
Is it possible to ever get a list of more than one
epoger
2014/07/15 13:20:18
AFAICT we should always get either 0 or 1 matching
|
+ acls.entries.entry_list.remove(entry) |
+ |
+ # Add a new entry to the ACLs. |
+ args = {'type': id_type, 'permission': permission} |
+ args[field] = id_value |
+ new_entry = acl.Entry(**args) |
+ acls.entries.entry_list.append(new_entry) |
+ b.set_acl(acl_or_str=acls, key_name=path) |
+ |
+ def delete_acl(self, bucket, path, id_type, id_value): |
+ """Delete certain access permissions on a single file in Google Storage. |
+ |
+ Various users who match this id_type/id_value pair may still have access |
+ rights to this file after this call, if they have been granted those rights |
+ based on *other* id_types (e.g., perhaps they still have individual user |
+ access rights, even if their group access rights are removed). |
+ |
+ If no permissions have been added for this id_type/id_value, this will |
+ return uneventfully (there will be no exception or other indication of |
+ failure). |
+ |
+ Params: |
+ bucket: GS bucket |
+ path: full path (Posix-style) to the file within that bucket |
+ id_type: must be one of the ID_TYPE_* constants defined above |
+ id_value: delete permissions for users whose id_type field contains this |
+ value |
+ """ |
+ field = FIELD_BY_ID_TYPE[id_type] |
+ conn = self._create_connection() |
+ b = conn.get_bucket(bucket_name=bucket) |
+ acls = b.get_acl(key_name=path) |
+ matching_entries = [entry for entry in acls.entries.entry_list |
+ if (entry.scope.type == id_type) and |
+ (getattr(entry.scope, field) == id_value)] |
+ if matching_entries: |
+ for entry in matching_entries: |
+ acls.entries.entry_list.remove(entry) |
+ b.set_acl(acl_or_str=acls, key_name=path) |
+ |
+ def get_acl(self, bucket, path, id_type, id_value): |
+ """Retrieve partial access permissions on a single file in Google Storage. |
+ |
+ Various users who match this id_type/id_value pair may have access rights |
+ other than that returned by this call, if they have been granted those |
+ rights based on *other* id_types (e.g., perhaps they have group access |
+ rights, beyond their individual access rights). |
+ |
+ Params: |
+ bucket: GS bucket |
+ path: full path (Posix-style) to the file within that bucket |
+ id_type: must be one of the ID_TYPE_* constants defined above |
+ id_value: delete permissions for users whose id_type field contains this |
epoger
2014/07/15 13:34:26
delete -> get
|
+ value |
+ |
epoger
2014/07/15 13:34:26
Add an example
|
+ Returns: the PERMISSION_* constant which has been set for users matching |
+ this id_type/id_value, on this file; or None if no such permissions have |
+ been set. |
rmistry
2014/07/15 11:26:10
[Optional] How about creating a PERMISSION_NONE or
epoger
2014/07/15 13:20:18
I can do that, *if* you think that would also make
|
+ """ |
+ field = FIELD_BY_ID_TYPE[id_type] |
+ conn = self._create_connection() |
+ b = conn.get_bucket(bucket_name=bucket) |
+ acls = b.get_acl(key_name=path) |
+ matching_entries = [entry for entry in acls.entries.entry_list |
+ if (entry.scope.type == id_type) and |
+ (getattr(entry.scope, field) == id_value)] |
+ if matching_entries: |
+ assert len(matching_entries) == 1, '%d == 1' % len(matching_entries) |
+ return matching_entries[0].permission |
+ else: |
+ return None |
+ |
def list_bucket_contents(self, bucket, subdir=None): |
"""Returns files in the Google Storage bucket as a (dirs, files) tuple. |
@@ -214,6 +344,39 @@ def _run_self_test(): |
assert dirs == [], '%s == []' % dirs |
assert files == filenames_to_upload, '%s == %s' % (files, filenames_to_upload) |
+ # Manipulate ACLs on one of those files, and verify them. |
+ # TODO(epoger): Test id_types other than ID_TYPE_GROUP_BY_DOMAIN ? |
+ # TODO(epoger): Test setting multiple ACLs on the same file? |
+ id_type = ID_TYPE_GROUP_BY_DOMAIN |
+ id_value = 'google.com' |
+ fullpath = posixpath.join(remote_dir, subdir, filenames_to_upload[0]) |
+ # Make sure ACL is empty to start with ... |
+ gs.delete_acl(bucket=bucket, path=fullpath, |
+ id_type=id_type, id_value=id_value) |
+ permission = gs.get_acl(bucket=bucket, path=fullpath, |
+ id_type=id_type, id_value=id_value) |
+ assert permission == None, '%s == None' % permission |
+ # ... set it to OWNER ... |
+ gs.add_acl(bucket=bucket, path=fullpath, |
+ id_type=id_type, id_value=id_value, permission=PERMISSION_OWNER) |
+ permission = gs.get_acl(bucket=bucket, path=fullpath, |
+ id_type=id_type, id_value=id_value) |
+ assert permission == PERMISSION_OWNER, '%s == %s' % ( |
+ permission, PERMISSION_OWNER) |
+ # ... now set it to READ ... |
+ gs.add_acl(bucket=bucket, path=fullpath, |
+ id_type=id_type, id_value=id_value, permission=PERMISSION_READ) |
+ permission = gs.get_acl(bucket=bucket, path=fullpath, |
+ id_type=id_type, id_value=id_value) |
+ assert permission == PERMISSION_READ, '%s == %s' % ( |
+ permission, PERMISSION_READ) |
+ # ... and clear it again to finish. |
+ gs.delete_acl(bucket=bucket, path=fullpath, |
+ id_type=id_type, id_value=id_value) |
+ permission = gs.get_acl(bucket=bucket, path=fullpath, |
+ id_type=id_type, id_value=id_value) |
+ assert permission == None, '%s == None' % permission |
+ |
# Download the files we uploaded to Google Storage, and validate contents. |
local_dest_dir = tempfile.mkdtemp() |
try: |