OLD | NEW |
---|---|
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 # pylint: disable=C0301 | 3 # pylint: disable=C0301 |
4 """ | 4 """ |
5 Copyright 2014 Google Inc. | 5 Copyright 2014 Google Inc. |
6 | 6 |
7 Use of this source code is governed by a BSD-style license that can be | 7 Use of this source code is governed by a BSD-style license that can be |
8 found in the LICENSE file. | 8 found in the LICENSE file. |
9 | 9 |
10 Utilities for accessing Google Cloud Storage, using the boto library (wrapper | 10 Utilities for accessing Google Cloud Storage, using the boto library (wrapper |
11 for the XML API). | 11 for the XML API). |
12 | 12 |
13 API/library references: | 13 API/library references: |
14 - https://developers.google.com/storage/docs/reference-guide | 14 - https://developers.google.com/storage/docs/reference-guide |
15 - http://googlecloudstorage.blogspot.com/2012/09/google-cloud-storage-tutorial-u sing-boto.html | 15 - http://googlecloudstorage.blogspot.com/2012/09/google-cloud-storage-tutorial-u sing-boto.html |
16 """ | 16 """ |
17 # pylint: enable=C0301 | 17 # pylint: enable=C0301 |
18 | 18 |
19 # System-level imports | 19 # System-level imports |
20 import errno | 20 import errno |
21 import os | 21 import os |
22 import posixpath | 22 import posixpath |
23 import random | |
24 import re | 23 import re |
25 import shutil | |
26 import sys | 24 import sys |
27 import tempfile | |
28 | 25 |
29 # Imports from third-party code | 26 # Imports from third-party code |
30 TRUNK_DIRECTORY = os.path.abspath(os.path.join( | 27 TRUNK_DIRECTORY = os.path.abspath(os.path.join( |
31 os.path.dirname(__file__), os.pardir, os.pardir)) | 28 os.path.dirname(__file__), os.pardir, os.pardir)) |
32 for import_subdir in ['boto']: | 29 for import_subdir in ['boto']: |
33 import_dirpath = os.path.join( | 30 import_dirpath = os.path.join( |
34 TRUNK_DIRECTORY, 'third_party', 'externals', import_subdir) | 31 TRUNK_DIRECTORY, 'third_party', 'externals', import_subdir) |
35 if import_dirpath not in sys.path: | 32 if import_dirpath not in sys.path: |
36 # We need to insert at the beginning of the path, to make sure that our | 33 # We need to insert at the beginning of the path, to make sure that our |
37 # imported versions are favored over others that might be in the path. | 34 # imported versions are favored over others that might be in the path. |
38 sys.path.insert(0, import_dirpath) | 35 sys.path.insert(0, import_dirpath) |
39 from boto.exception import BotoServerError | 36 from boto.exception import BotoServerError |
40 from boto.gs import acl | 37 from boto.gs import acl |
41 from boto.gs.bucket import Bucket | 38 from boto.gs.bucket import Bucket |
42 from boto.gs.connection import GSConnection | 39 from boto.gs.connection import GSConnection |
43 from boto.gs.key import Key | 40 from boto.gs.key import Key |
44 from boto.s3.bucketlistresultset import BucketListResultSet | 41 from boto.s3.bucketlistresultset import BucketListResultSet |
45 from boto.s3.connection import SubdomainCallingFormat | 42 from boto.s3.connection import SubdomainCallingFormat |
46 from boto.s3.prefix import Prefix | 43 from boto.s3.prefix import Prefix |
47 | 44 |
48 # Predefined (aka "canned") ACLs that provide a "base coat" of permissions for | |
49 # each file in Google Storage. See CannedACLStrings in | |
50 # https://github.com/boto/boto/blob/develop/boto/gs/acl.py | |
51 # Also see https://developers.google.com/storage/docs/accesscontrol | |
52 PREDEFINED_ACL_AUTHENTICATED_READ = 'authenticated-read' | |
53 PREDEFINED_ACL_BUCKET_OWNER_FULL_CONTROL = 'bucket-owner-full-control' | |
54 PREDEFINED_ACL_BUCKET_OWNER_READ = 'bucket-owner-read' | |
55 PREDEFINED_ACL_PRIVATE = 'private' | |
56 PREDEFINED_ACL_PROJECT_PRIVATE = 'project-private' | |
57 PREDEFINED_ACL_PUBLIC_READ = 'public-read' | |
58 PREDEFINED_ACL_PUBLIC_READ_WRITE = 'public-read-write' | |
59 | |
60 # "Fine-grained" permissions that may be set per user/group on each file in | |
61 # Google Storage. See SupportedPermissions in | |
62 # https://github.com/boto/boto/blob/develop/boto/gs/acl.py | |
63 # Also see https://developers.google.com/storage/docs/accesscontrol | |
64 PERMISSION_NONE = None | |
65 PERMISSION_OWNER = 'FULL_CONTROL' | |
66 PERMISSION_READ = 'READ' | |
67 PERMISSION_WRITE = 'WRITE' | |
68 | |
69 # Types of identifiers we can use to set "fine-grained" ACLs. | |
70 ID_TYPE_GROUP_BY_DOMAIN = acl.GROUP_BY_DOMAIN | |
71 ID_TYPE_GROUP_BY_EMAIL = acl.GROUP_BY_EMAIL | |
72 ID_TYPE_GROUP_BY_ID = acl.GROUP_BY_ID | |
73 ID_TYPE_USER_BY_EMAIL = acl.USER_BY_EMAIL | |
74 ID_TYPE_USER_BY_ID = acl.USER_BY_ID | |
75 | |
76 # Which field we get/set in ACL entries, depending on ID_TYPE. | |
77 FIELD_BY_ID_TYPE = { | |
78 ID_TYPE_GROUP_BY_DOMAIN: 'domain', | |
79 ID_TYPE_GROUP_BY_EMAIL: 'email_address', | |
80 ID_TYPE_GROUP_BY_ID: 'id', | |
81 ID_TYPE_USER_BY_EMAIL: 'email_address', | |
82 ID_TYPE_USER_BY_ID: 'id', | |
83 } | |
84 | |
85 | 45 |
86 class AnonymousGSConnection(GSConnection): | 46 class AnonymousGSConnection(GSConnection): |
87 """GSConnection class that allows anonymous connections. | 47 """GSConnection class that allows anonymous connections. |
88 | 48 |
89 The GSConnection class constructor in | 49 The GSConnection class constructor in |
90 https://github.com/boto/boto/blob/develop/boto/gs/connection.py doesn't allow | 50 https://github.com/boto/boto/blob/develop/boto/gs/connection.py doesn't allow |
91 for anonymous connections (connections without credentials), so we have to | 51 for anonymous connections (connections without credentials), so we have to |
92 override it. | 52 override it. |
93 """ | 53 """ |
94 def __init__(self): | 54 def __init__(self): |
95 super(GSConnection, self).__init__( | 55 super(GSConnection, self).__init__( |
96 # This is the important bit we need to add... | 56 # This is the important bit we need to add... |
97 anon=True, | 57 anon=True, |
98 # ...and these are just copied in from GSConnection.__init__() | 58 # ...and these are just copied in from GSConnection.__init__() |
99 bucket_class=Bucket, | 59 bucket_class=Bucket, |
100 calling_format=SubdomainCallingFormat(), | 60 calling_format=SubdomainCallingFormat(), |
101 host=GSConnection.DefaultHost, | 61 host=GSConnection.DefaultHost, |
102 provider='google') | 62 provider='google') |
103 | 63 |
104 | 64 |
105 class GSUtils(object): | 65 class GSUtils(object): |
106 """Utilities for accessing Google Cloud Storage, using the boto library.""" | 66 """Utilities for accessing Google Cloud Storage, using the boto library.""" |
107 | 67 |
68 class Permission: | |
69 """Fine-grained permissions that may be set per user/group on each file. | |
70 | |
71 See SupportedPermissions in | |
72 https://github.com/boto/boto/blob/develop/boto/gs/acl.py | |
73 Also see https://developers.google.com/storage/docs/accesscontrol | |
74 """ | |
75 Empty = None | |
borenet
2014/07/21 17:35:25
Should these be in all-caps, since they're constan
epoger
2014/07/21 18:03:19
SURE_WHY_NOT
| |
76 Owner = 'FULL_CONTROL' | |
77 Read = 'READ' | |
78 Write = 'WRITE' | |
79 | |
80 class PredefinedACL: | |
81 """Canned ACLs that provide a "base coat" of permissions for each file. | |
82 | |
83 See CannedACLStrings in | |
84 https://github.com/boto/boto/blob/develop/boto/gs/acl.py | |
85 Also see https://developers.google.com/storage/docs/accesscontrol | |
86 """ | |
87 AuthenticatedRead = 'authenticated-read' | |
88 BucketOwnerFullControl = 'bucket-owner-full-control' | |
89 BucketOwnerRead = 'bucket-owner-read' | |
90 Private = 'private' | |
91 ProjectPrivate = 'project-private' | |
92 PublicRead = 'public-read' | |
93 PublicReadWrite = 'public-read-write' | |
94 | |
95 class IdType: | |
96 """Types of identifiers we can use to set "fine-grained" ACLs.""" | |
97 GroupByDomain = acl.GROUP_BY_DOMAIN | |
98 GroupByEmail = acl.GROUP_BY_EMAIL | |
99 GroupById = acl.GROUP_BY_ID | |
100 UserByEmail = acl.USER_BY_EMAIL | |
101 UserById = acl.USER_BY_ID | |
102 | |
103 | |
108 def __init__(self, boto_file_path=None): | 104 def __init__(self, boto_file_path=None): |
109 """Constructor. | 105 """Constructor. |
110 | 106 |
111 Params: | 107 Params: |
112 boto_file_path: full path (local-OS-style) on local disk where .boto | 108 boto_file_path: full path (local-OS-style) on local disk where .boto |
113 credentials file can be found. If None, then the GSUtils object | 109 credentials file can be found. If None, then the GSUtils object |
114 created will be able to access only public files in Google Storage. | 110 created will be able to access only public files in Google Storage. |
115 | 111 |
116 Raises an exception if no file is found at boto_file_path, or if the file | 112 Raises an exception if no file is found at boto_file_path, or if the file |
117 found there is malformed. | 113 found there is malformed. |
118 """ | 114 """ |
119 self._gs_access_key_id = None | 115 self._gs_access_key_id = None |
120 self._gs_secret_access_key = None | 116 self._gs_secret_access_key = None |
121 if boto_file_path: | 117 if boto_file_path: |
122 print 'Reading boto file from %s' % boto_file_path | 118 print 'Reading boto file from %s' % boto_file_path |
123 boto_dict = _config_file_as_dict(filepath=boto_file_path) | 119 boto_dict = _config_file_as_dict(filepath=boto_file_path) |
124 self._gs_access_key_id = boto_dict['gs_access_key_id'] | 120 self._gs_access_key_id = boto_dict['gs_access_key_id'] |
125 self._gs_secret_access_key = boto_dict['gs_secret_access_key'] | 121 self._gs_secret_access_key = boto_dict['gs_secret_access_key'] |
122 # Which field we get/set in ACL entries, depending on IdType. | |
123 self._field_by_id_type = { | |
124 self.IdType.GroupByDomain: 'domain', | |
125 self.IdType.GroupByEmail: 'email_address', | |
126 self.IdType.GroupById: 'id', | |
127 self.IdType.UserByEmail: 'email_address', | |
128 self.IdType.UserById: 'id', | |
129 } | |
126 | 130 |
127 def delete_file(self, bucket, path): | 131 def delete_file(self, bucket, path): |
128 """Delete a single file within a GS bucket. | 132 """Delete a single file within a GS bucket. |
129 | 133 |
130 TODO(epoger): what if bucket or path does not exist? Should probably raise | 134 TODO(epoger): what if bucket or path does not exist? Should probably raise |
131 an exception. Implement, and add a test to exercise this. | 135 an exception. Implement, and add a test to exercise this. |
132 | 136 |
133 Params: | 137 Params: |
134 bucket: GS bucket to delete a file from | 138 bucket: GS bucket to delete a file from |
135 path: full path (Posix-style) of the file within the bucket to delete | 139 path: full path (Posix-style) of the file within the bucket to delete |
(...skipping 14 matching lines...) Expand all Loading... | |
150 | 154 |
151 TODO(epoger): Add the only_if_modified param provided by upload_file() in | 155 TODO(epoger): Add the only_if_modified param provided by upload_file() in |
152 https://github.com/google/skia-buildbot/blob/master/slave/skia_slave_scripts /utils/old_gs_utils.py , | 156 https://github.com/google/skia-buildbot/blob/master/slave/skia_slave_scripts /utils/old_gs_utils.py , |
153 so we can replace that function with this one. | 157 so we can replace that function with this one. |
154 | 158 |
155 params: | 159 params: |
156 source_path: full path (local-OS-style) on local disk to read from | 160 source_path: full path (local-OS-style) on local disk to read from |
157 dest_bucket: GCS bucket to copy the file to | 161 dest_bucket: GCS bucket to copy the file to |
158 dest_path: full path (Posix-style) within that bucket | 162 dest_path: full path (Posix-style) within that bucket |
159 predefined_acl: which predefined ACL to apply to the file on Google | 163 predefined_acl: which predefined ACL to apply to the file on Google |
160 Storage; must be one of the PREDEFINED_ACL_* constants defined above. | 164 Storage; must be one of the PredefinedACL values defined above. |
161 If None, inherits dest_bucket's default object ACL. | 165 If None, inherits dest_bucket's default object ACL. |
162 TODO(epoger): add unittests for this param, although it seems to work | 166 TODO(epoger): add unittests for this param, although it seems to work |
163 in my manual testing | 167 in my manual testing |
164 fine_grained_acl_list: list of (id_type, id_value, permission) tuples | 168 fine_grained_acl_list: list of (id_type, id_value, permission) tuples |
165 to apply to the uploaded file (on top of the predefined_acl), | 169 to apply to the uploaded file (on top of the predefined_acl), |
166 or None if predefined_acl is sufficient | 170 or None if predefined_acl is sufficient |
167 """ | 171 """ |
168 b = self._connect_to_bucket(bucket_name=dest_bucket) | 172 b = self._connect_to_bucket(bucket_name=dest_bucket) |
169 item = Key(b) | 173 item = Key(b) |
170 item.key = dest_path | 174 item.key = dest_path |
(...skipping 17 matching lines...) Expand all Loading... | |
188 predefined_acl=None, fine_grained_acl_list=None): | 192 predefined_acl=None, fine_grained_acl_list=None): |
189 """Recursively upload contents of a local directory to Google Storage. | 193 """Recursively upload contents of a local directory to Google Storage. |
190 | 194 |
191 params: | 195 params: |
192 source_dir: full path (local-OS-style) on local disk of directory to copy | 196 source_dir: full path (local-OS-style) on local disk of directory to copy |
193 contents of | 197 contents of |
194 dest_bucket: GCS bucket to copy the files into | 198 dest_bucket: GCS bucket to copy the files into |
195 dest_dir: full path (Posix-style) within that bucket; write the files into | 199 dest_dir: full path (Posix-style) within that bucket; write the files into |
196 this directory | 200 this directory |
197 predefined_acl: which predefined ACL to apply to the files on Google | 201 predefined_acl: which predefined ACL to apply to the files on Google |
198 Storage; must be one of the PREDEFINED_ACL_* constants defined above. | 202 Storage; must be one of the PredefinedACL values defined above. |
199 If None, inherits dest_bucket's default object ACL. | 203 If None, inherits dest_bucket's default object ACL. |
200 TODO(epoger): add unittests for this param, although it seems to work | 204 TODO(epoger): add unittests for this param, although it seems to work |
201 in my manual testing | 205 in my manual testing |
202 fine_grained_acl_list: list of (id_type, id_value, permission) tuples | 206 fine_grained_acl_list: list of (id_type, id_value, permission) tuples |
203 to apply to every file uploaded (on top of the predefined_acl), | 207 to apply to every file uploaded (on top of the predefined_acl), |
204 or None if predefined_acl is sufficient | 208 or None if predefined_acl is sufficient |
205 | 209 |
206 The copy operates as a "merge with overwrite": any files in source_dir will | 210 The copy operates as a "merge with overwrite": any files in source_dir will |
207 be "overlaid" on top of the existing content in dest_dir. Existing files | 211 be "overlaid" on top of the existing content in dest_dir. Existing files |
208 with the same names will be overwritten. | 212 with the same names will be overwritten. |
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
319 other than that returned by this call, if they have been granted those | 323 other than that returned by this call, if they have been granted those |
320 rights based on *other* id_types (e.g., perhaps they have group access | 324 rights based on *other* id_types (e.g., perhaps they have group access |
321 rights, beyond their individual access rights). | 325 rights, beyond their individual access rights). |
322 | 326 |
323 TODO(epoger): What if the remote file does not exist? This should probably | 327 TODO(epoger): What if the remote file does not exist? This should probably |
324 raise an exception in that case. | 328 raise an exception in that case. |
325 | 329 |
326 Params: | 330 Params: |
327 bucket: GS bucket | 331 bucket: GS bucket |
328 path: full path (Posix-style) to the file within that bucket | 332 path: full path (Posix-style) to the file within that bucket |
329 id_type: must be one of the ID_TYPE_* constants defined above | 333 id_type: must be one of the IdType values defined above |
330 id_value: get permissions for users whose id_type field contains this | 334 id_value: get permissions for users whose id_type field contains this |
331 value | 335 value |
332 | 336 |
333 Returns: the PERMISSION_* constant which has been set for users matching | 337 Returns: the Permission value which has been set for users matching |
334 this id_type/id_value, on this file; or PERMISSION_NONE if no such | 338 this id_type/id_value, on this file; or Permission.Empty if no such |
335 permissions have been set. | 339 permissions have been set. |
336 """ | 340 """ |
337 field = FIELD_BY_ID_TYPE[id_type] | 341 field = self._field_by_id_type[id_type] |
338 b = self._connect_to_bucket(bucket_name=bucket) | 342 b = self._connect_to_bucket(bucket_name=bucket) |
339 acls = b.get_acl(key_name=path) | 343 acls = b.get_acl(key_name=path) |
340 matching_entries = [entry for entry in acls.entries.entry_list | 344 matching_entries = [entry for entry in acls.entries.entry_list |
341 if (entry.scope.type == id_type) and | 345 if (entry.scope.type == id_type) and |
342 (getattr(entry.scope, field) == id_value)] | 346 (getattr(entry.scope, field) == id_value)] |
343 if matching_entries: | 347 if matching_entries: |
344 assert len(matching_entries) == 1, '%d == 1' % len(matching_entries) | 348 assert len(matching_entries) == 1, '%d == 1' % len(matching_entries) |
345 return matching_entries[0].permission | 349 return matching_entries[0].permission |
346 else: | 350 else: |
347 return PERMISSION_NONE | 351 return self.Permission.Empty |
348 | 352 |
349 def set_acl(self, bucket, path, id_type, id_value, permission): | 353 def set_acl(self, bucket, path, id_type, id_value, permission): |
350 """Set partial access permissions on a single file in Google Storage. | 354 """Set partial access permissions on a single file in Google Storage. |
351 | 355 |
352 Note that a single set_acl() call will not guarantee what access rights any | 356 Note that a single set_acl() call will not guarantee what access rights any |
353 given user will have on a given file, because permissions are additive. | 357 given user will have on a given file, because permissions are additive. |
354 (E.g., if you set READ permission for a group, but a member of that group | 358 (E.g., if you set READ permission for a group, but a member of that group |
355 already has WRITE permission, that member will still have WRITE permission.) | 359 already has WRITE permission, that member will still have WRITE permission.) |
356 TODO(epoger): Do we know that for sure? I *think* that's how it works... | 360 TODO(epoger): Do we know that for sure? I *think* that's how it works... |
357 | 361 |
358 If there is already a permission set on this file for this id_type/id_value | 362 If there is already a permission set on this file for this id_type/id_value |
359 combination, this call will overwrite it. | 363 combination, this call will overwrite it. |
360 | 364 |
361 TODO(epoger): What if the remote file does not exist? This should probably | 365 TODO(epoger): What if the remote file does not exist? This should probably |
362 raise an exception in that case. | 366 raise an exception in that case. |
363 | 367 |
364 Params: | 368 Params: |
365 bucket: GS bucket | 369 bucket: GS bucket |
366 path: full path (Posix-style) to the file within that bucket | 370 path: full path (Posix-style) to the file within that bucket |
367 id_type: must be one of the ID_TYPE_* constants defined above | 371 id_type: must be one of the IdType values defined above |
368 id_value: add permission for users whose id_type field contains this value | 372 id_value: add permission for users whose id_type field contains this value |
369 permission: permission to add for users matching id_type/id_value; | 373 permission: permission to add for users matching id_type/id_value; |
370 must be one of the PERMISSION_* constants defined above. | 374 must be one of the Permission values defined above. |
371 If PERMISSION_NONE, then any permissions will be granted to this | 375 If Permission.Empty, then any permissions will be granted to this |
372 particular id_type/id_value will be removed... but, given that | 376 particular id_type/id_value will be removed... but, given that |
373 permissions are additive, specific users may still have access rights | 377 permissions are additive, specific users may still have access rights |
374 based on permissions given to *other* id_type/id_value pairs. | 378 based on permissions given to *other* id_type/id_value pairs. |
375 | 379 |
376 Example Code: | 380 Example Code: |
377 bucket = 'gs://bucket-name' | 381 bucket = 'gs://bucket-name' |
378 path = 'path/to/file' | 382 path = 'path/to/file' |
379 id_type = ID_TYPE_USER_BY_EMAIL | 383 id_type = IdType.UserByEmail |
380 id_value = 'epoger@google.com' | 384 id_value = 'epoger@google.com' |
381 set_acl(bucket, path, id_type, id_value, PERMISSION_READ) | 385 set_acl(bucket, path, id_type, id_value, Permission.Read) |
382 assert PERMISSION_READ == get_acl(bucket, path, id_type, id_value) | 386 assert Permission.Read == get_acl(bucket, path, id_type, id_value) |
383 set_acl(bucket, path, id_type, id_value, PERMISSION_WRITE) | 387 set_acl(bucket, path, id_type, id_value, Permission.Write) |
384 assert PERMISSION_WRITE == get_acl(bucket, path, id_type, id_value) | 388 assert Permission.Write == get_acl(bucket, path, id_type, id_value) |
385 """ | 389 """ |
386 field = FIELD_BY_ID_TYPE[id_type] | 390 field = self._field_by_id_type[id_type] |
387 b = self._connect_to_bucket(bucket_name=bucket) | 391 b = self._connect_to_bucket(bucket_name=bucket) |
388 acls = b.get_acl(key_name=path) | 392 acls = b.get_acl(key_name=path) |
389 | 393 |
390 # Remove any existing entries that refer to the same id_type/id_value, | 394 # Remove any existing entries that refer to the same id_type/id_value, |
391 # because the API will fail if we try to set more than one. | 395 # because the API will fail if we try to set more than one. |
392 matching_entries = [entry for entry in acls.entries.entry_list | 396 matching_entries = [entry for entry in acls.entries.entry_list |
393 if (entry.scope.type == id_type) and | 397 if (entry.scope.type == id_type) and |
394 (getattr(entry.scope, field) == id_value)] | 398 (getattr(entry.scope, field) == id_value)] |
395 if matching_entries: | 399 if matching_entries: |
396 assert len(matching_entries) == 1, '%d == 1' % len(matching_entries) | 400 assert len(matching_entries) == 1, '%d == 1' % len(matching_entries) |
397 acls.entries.entry_list.remove(matching_entries[0]) | 401 acls.entries.entry_list.remove(matching_entries[0]) |
398 | 402 |
399 # Add a new entry to the ACLs. | 403 # Add a new entry to the ACLs. |
400 if permission != PERMISSION_NONE: | 404 if permission != self.Permission.Empty: |
401 args = {'type': id_type, 'permission': permission} | 405 args = {'type': id_type, 'permission': permission} |
402 args[field] = id_value | 406 args[field] = id_value |
403 new_entry = acl.Entry(**args) | 407 new_entry = acl.Entry(**args) |
404 acls.entries.entry_list.append(new_entry) | 408 acls.entries.entry_list.append(new_entry) |
405 | 409 |
406 # Finally, write back the modified ACLs. | 410 # Finally, write back the modified ACLs. |
407 b.set_acl(acl_or_str=acls, key_name=path) | 411 b.set_acl(acl_or_str=acls, key_name=path) |
408 | 412 |
409 def list_bucket_contents(self, bucket, subdir=None): | 413 def list_bucket_contents(self, bucket, subdir=None): |
410 """Returns files in the Google Storage bucket as a (dirs, files) tuple. | 414 """Returns files in the Google Storage bucket as a (dirs, files) tuple. |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
448 | 452 |
449 def _create_connection(self): | 453 def _create_connection(self): |
450 """Returns a GSConnection object we can use to access Google Storage.""" | 454 """Returns a GSConnection object we can use to access Google Storage.""" |
451 if self._gs_access_key_id: | 455 if self._gs_access_key_id: |
452 return GSConnection( | 456 return GSConnection( |
453 gs_access_key_id=self._gs_access_key_id, | 457 gs_access_key_id=self._gs_access_key_id, |
454 gs_secret_access_key=self._gs_secret_access_key) | 458 gs_secret_access_key=self._gs_secret_access_key) |
455 else: | 459 else: |
456 return AnonymousGSConnection() | 460 return AnonymousGSConnection() |
457 | 461 |
462 | |
458 def _config_file_as_dict(filepath): | 463 def _config_file_as_dict(filepath): |
459 """Reads a boto-style config file into a dict. | 464 """Reads a boto-style config file into a dict. |
460 | 465 |
461 Parses all lines from the file of this form: key = value | 466 Parses all lines from the file of this form: key = value |
462 TODO(epoger): Create unittest. | 467 TODO(epoger): Create unittest. |
463 | 468 |
464 Params: | 469 Params: |
465 filepath: path to config file on local disk | 470 filepath: path to config file on local disk |
466 | 471 |
467 Returns: contents of the config file, as a dictionary | 472 Returns: contents of the config file, as a dictionary |
(...skipping 16 matching lines...) Expand all Loading... | |
484 exist yet. | 489 exist yet. |
485 | 490 |
486 Args: | 491 Args: |
487 path: full path of directory to create | 492 path: full path of directory to create |
488 """ | 493 """ |
489 try: | 494 try: |
490 os.makedirs(path) | 495 os.makedirs(path) |
491 except OSError as e: | 496 except OSError as e: |
492 if e.errno != errno.EEXIST: | 497 if e.errno != errno.EEXIST: |
493 raise | 498 raise |
494 | |
495 | |
496 def _test_public_read(): | |
497 """Make sure we can read from public files without .boto file credentials.""" | |
498 gs = GSUtils() | |
499 gs.list_bucket_contents(bucket='chromium-skia-gm-summaries', subdir=None) | |
500 | |
501 | |
502 def _test_authenticated_round_trip(): | |
503 try: | |
504 gs = GSUtils(boto_file_path=os.path.expanduser(os.path.join('~','.boto'))) | |
505 except: | |
506 print """ | |
507 Failed to instantiate GSUtils object with default .boto file path. | |
508 Do you have a ~/.boto file that provides the credentials needed to read | |
509 and write gs://chromium-skia-gm ? | |
510 """ | |
511 raise | |
512 | |
513 bucket = 'chromium-skia-gm' | |
514 remote_dir = 'gs_utils_test/%d' % random.randint(0, sys.maxint) | |
515 subdir = 'subdir' | |
516 filenames_to_upload = ['file1', 'file2'] | |
517 | |
518 # Upload test files to Google Storage, checking that their fine-grained | |
519 # ACLs were set correctly. | |
520 id_type = ID_TYPE_GROUP_BY_DOMAIN | |
521 id_value = 'chromium.org' | |
522 set_permission = PERMISSION_READ | |
523 local_src_dir = tempfile.mkdtemp() | |
524 os.mkdir(os.path.join(local_src_dir, subdir)) | |
525 try: | |
526 for filename in filenames_to_upload: | |
527 with open(os.path.join(local_src_dir, subdir, filename), 'w') as f: | |
528 f.write('contents of %s\n' % filename) | |
529 dest_path = posixpath.join(remote_dir, subdir, filename) | |
530 gs.upload_file( | |
531 source_path=os.path.join(local_src_dir, subdir, filename), | |
532 dest_bucket=bucket, dest_path=dest_path, | |
533 fine_grained_acl_list=[(id_type, id_value, set_permission)]) | |
534 got_permission = gs.get_acl(bucket=bucket, path=dest_path, | |
535 id_type=id_type, id_value=id_value) | |
536 assert got_permission == set_permission, '%s == %s' % ( | |
537 got_permission, set_permission) | |
538 finally: | |
539 shutil.rmtree(local_src_dir) | |
540 | |
541 # Get a list of the files we uploaded to Google Storage. | |
542 (dirs, files) = gs.list_bucket_contents( | |
543 bucket=bucket, subdir=remote_dir) | |
544 assert dirs == [subdir], '%s == [%s]' % (dirs, subdir) | |
545 assert files == [], '%s == []' % files | |
546 (dirs, files) = gs.list_bucket_contents( | |
547 bucket=bucket, subdir=posixpath.join(remote_dir, subdir)) | |
548 assert dirs == [], '%s == []' % dirs | |
549 assert files == filenames_to_upload, '%s == %s' % (files, filenames_to_upload) | |
550 | |
551 # Manipulate ACLs on one of those files, and verify them. | |
552 # TODO(epoger): Test id_types other than ID_TYPE_GROUP_BY_DOMAIN ? | |
553 # TODO(epoger): Test setting multiple ACLs on the same file? | |
554 id_type = ID_TYPE_GROUP_BY_DOMAIN | |
555 id_value = 'google.com' | |
556 fullpath = posixpath.join(remote_dir, subdir, filenames_to_upload[0]) | |
557 # Make sure ACL is empty to start with ... | |
558 gs.set_acl(bucket=bucket, path=fullpath, | |
559 id_type=id_type, id_value=id_value, permission=PERMISSION_NONE) | |
560 permission = gs.get_acl(bucket=bucket, path=fullpath, | |
561 id_type=id_type, id_value=id_value) | |
562 assert permission == PERMISSION_NONE, '%s == %s' % ( | |
563 permission, PERMISSION_NONE) | |
564 # ... set it to OWNER ... | |
565 gs.set_acl(bucket=bucket, path=fullpath, | |
566 id_type=id_type, id_value=id_value, permission=PERMISSION_OWNER) | |
567 permission = gs.get_acl(bucket=bucket, path=fullpath, | |
568 id_type=id_type, id_value=id_value) | |
569 assert permission == PERMISSION_OWNER, '%s == %s' % ( | |
570 permission, PERMISSION_OWNER) | |
571 # ... now set it to READ ... | |
572 gs.set_acl(bucket=bucket, path=fullpath, | |
573 id_type=id_type, id_value=id_value, permission=PERMISSION_READ) | |
574 permission = gs.get_acl(bucket=bucket, path=fullpath, | |
575 id_type=id_type, id_value=id_value) | |
576 assert permission == PERMISSION_READ, '%s == %s' % ( | |
577 permission, PERMISSION_READ) | |
578 # ... and clear it again to finish. | |
579 gs.set_acl(bucket=bucket, path=fullpath, | |
580 id_type=id_type, id_value=id_value, permission=PERMISSION_NONE) | |
581 permission = gs.get_acl(bucket=bucket, path=fullpath, | |
582 id_type=id_type, id_value=id_value) | |
583 assert permission == PERMISSION_NONE, '%s == %s' % ( | |
584 permission, PERMISSION_NONE) | |
585 | |
586 # Download the files we uploaded to Google Storage, and validate contents. | |
587 local_dest_dir = tempfile.mkdtemp() | |
588 try: | |
589 for filename in filenames_to_upload: | |
590 gs.download_file(source_bucket=bucket, | |
591 source_path=posixpath.join(remote_dir, subdir, filename), | |
592 dest_path=os.path.join(local_dest_dir, subdir, filename), | |
593 create_subdirs_if_needed=True) | |
594 with open(os.path.join(local_dest_dir, subdir, filename)) as f: | |
595 file_contents = f.read() | |
596 assert file_contents == 'contents of %s\n' % filename, ( | |
597 '%s == "contents of %s\n"' % (file_contents, filename)) | |
598 finally: | |
599 shutil.rmtree(local_dest_dir) | |
600 | |
601 # Delete all the files we uploaded to Google Storage. | |
602 for filename in filenames_to_upload: | |
603 gs.delete_file(bucket=bucket, | |
604 path=posixpath.join(remote_dir, subdir, filename)) | |
605 | |
606 # Confirm that we deleted all the files we uploaded to Google Storage. | |
607 (dirs, files) = gs.list_bucket_contents( | |
608 bucket=bucket, subdir=posixpath.join(remote_dir, subdir)) | |
609 assert dirs == [], '%s == []' % dirs | |
610 assert files == [], '%s == []' % files | |
611 | |
612 | |
613 def _test_dir_upload_and_download(): | |
614 """Test upload_dir_contents() and download_dir_contents().""" | |
615 try: | |
616 gs = GSUtils(boto_file_path=os.path.expanduser(os.path.join('~','.boto'))) | |
617 except: | |
618 print """ | |
619 Failed to instantiate GSUtils object with default .boto file path. | |
620 Do you have a ~/.boto file that provides the credentials needed to read | |
621 and write gs://chromium-skia-gm ? | |
622 """ | |
623 raise | |
624 | |
625 bucket = 'chromium-skia-gm' | |
626 remote_dir = 'gs_utils_test/%d' % random.randint(0, sys.maxint) | |
627 subdir = 'subdir' | |
628 filenames = ['file1', 'file2'] | |
629 | |
630 # Create directory tree on local disk and upload it. | |
631 id_type = ID_TYPE_GROUP_BY_DOMAIN | |
632 id_value = 'chromium.org' | |
633 set_permission = PERMISSION_READ | |
634 local_src_dir = tempfile.mkdtemp() | |
635 os.mkdir(os.path.join(local_src_dir, subdir)) | |
636 try: | |
637 for filename in filenames: | |
638 with open(os.path.join(local_src_dir, subdir, filename), 'w') as f: | |
639 f.write('contents of %s\n' % filename) | |
640 gs.upload_dir_contents( | |
641 source_dir=local_src_dir, dest_bucket=bucket, dest_dir=remote_dir, | |
642 predefined_acl=PREDEFINED_ACL_PRIVATE, | |
643 fine_grained_acl_list=[(id_type, id_value, set_permission)]) | |
644 finally: | |
645 shutil.rmtree(local_src_dir) | |
646 | |
647 # Validate the list of the files we uploaded to Google Storage. | |
648 (dirs, files) = gs.list_bucket_contents( | |
649 bucket=bucket, subdir=remote_dir) | |
650 assert dirs == [subdir], '%s == [%s]' % (dirs, subdir) | |
651 assert files == [], '%s == []' % files | |
652 (dirs, files) = gs.list_bucket_contents( | |
653 bucket=bucket, subdir=posixpath.join(remote_dir, subdir)) | |
654 assert dirs == [], '%s == []' % dirs | |
655 assert files == filenames, '%s == %s' % (files, filenames) | |
656 | |
657 # Check the fine-grained ACLs we set in Google Storage. | |
658 for filename in filenames: | |
659 got_permission = gs.get_acl( | |
660 bucket=bucket, path=posixpath.join(remote_dir, subdir, filename), | |
661 id_type=id_type, id_value=id_value) | |
662 assert got_permission == set_permission, '%s == %s' % ( | |
663 got_permission, set_permission) | |
664 | |
665 # Download the directory tree we just uploaded, make sure its contents | |
666 # are what we expect, and then delete the tree in Google Storage. | |
667 local_dest_dir = tempfile.mkdtemp() | |
668 try: | |
669 gs.download_dir_contents(source_bucket=bucket, source_dir=remote_dir, | |
670 dest_dir=local_dest_dir) | |
671 for filename in filenames: | |
672 with open(os.path.join(local_dest_dir, subdir, filename)) as f: | |
673 file_contents = f.read() | |
674 assert file_contents == 'contents of %s\n' % filename, ( | |
675 '%s == "contents of %s\n"' % (file_contents, filename)) | |
676 finally: | |
677 shutil.rmtree(local_dest_dir) | |
678 for filename in filenames: | |
679 gs.delete_file(bucket=bucket, | |
680 path=posixpath.join(remote_dir, subdir, filename)) | |
681 | |
682 | |
683 # TODO(epoger): How should we exercise these self-tests? | |
684 # See http://skbug.com/2751 | |
685 if __name__ == '__main__': | |
686 _test_public_read() | |
687 _test_authenticated_round_trip() | |
688 _test_dir_upload_and_download() | |
689 # TODO(epoger): Add _test_unauthenticated_access() to make sure we raise | |
690 # an exception when we try to access without needed credentials. | |
OLD | NEW |