| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- coding: utf-8 -*- | |
| 2 # Copyright 2014 Google Inc. All Rights Reserved. | |
| 3 # | |
| 4 # Licensed under the Apache License, Version 2.0 (the "License"); | |
| 5 # you may not use this file except in compliance with the License. | |
| 6 # You may obtain a copy of the License at | |
| 7 # | |
| 8 # http://www.apache.org/licenses/LICENSE-2.0 | |
| 9 # | |
| 10 # Unless required by applicable law or agreed to in writing, software | |
| 11 # distributed under the License is distributed on an "AS IS" BASIS, | |
| 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 13 # See the License for the specific language governing permissions and | |
| 14 # limitations under the License. | |
| 15 """Utility module for translating XML API objects to/from JSON objects.""" | |
| 16 | |
| 17 from __future__ import absolute_import | |
| 18 | |
| 19 import datetime | |
| 20 import json | |
| 21 import re | |
| 22 import textwrap | |
| 23 import xml.etree.ElementTree | |
| 24 | |
| 25 from apitools.base.py import encoding | |
| 26 import boto | |
| 27 from boto.gs.acl import ACL | |
| 28 from boto.gs.acl import ALL_AUTHENTICATED_USERS | |
| 29 from boto.gs.acl import ALL_USERS | |
| 30 from boto.gs.acl import Entries | |
| 31 from boto.gs.acl import Entry | |
| 32 from boto.gs.acl import GROUP_BY_DOMAIN | |
| 33 from boto.gs.acl import GROUP_BY_EMAIL | |
| 34 from boto.gs.acl import GROUP_BY_ID | |
| 35 from boto.gs.acl import USER_BY_EMAIL | |
| 36 from boto.gs.acl import USER_BY_ID | |
| 37 | |
| 38 from gslib.cloud_api import ArgumentException | |
| 39 from gslib.cloud_api import NotFoundException | |
| 40 from gslib.cloud_api import Preconditions | |
| 41 from gslib.exception import CommandException | |
| 42 from gslib.third_party.storage_apitools import storage_v1_messages as apitools_m
essages | |
| 43 | |
| 44 # In Python 2.6, ElementTree raises ExpatError instead of ParseError. | |
| 45 # pylint: disable=g-import-not-at-top | |
| 46 try: | |
| 47 from xml.etree.ElementTree import ParseError as XmlParseError | |
| 48 except ImportError: | |
| 49 from xml.parsers.expat import ExpatError as XmlParseError | |
| 50 | |
| 51 CACHE_CONTROL_REGEX = re.compile(r'^cache-control', re.I) | |
| 52 CONTENT_DISPOSITION_REGEX = re.compile(r'^content-disposition', re.I) | |
| 53 CONTENT_ENCODING_REGEX = re.compile(r'^content-encoding', re.I) | |
| 54 CONTENT_LANGUAGE_REGEX = re.compile(r'^content-language', re.I) | |
| 55 CONTENT_MD5_REGEX = re.compile(r'^content-md5', re.I) | |
| 56 CONTENT_TYPE_REGEX = re.compile(r'^content-type', re.I) | |
| 57 GOOG_API_VERSION_REGEX = re.compile(r'^x-goog-api-version', re.I) | |
| 58 GOOG_GENERATION_MATCH_REGEX = re.compile(r'^x-goog-if-generation-match', re.I) | |
| 59 GOOG_METAGENERATION_MATCH_REGEX = re.compile( | |
| 60 r'^x-goog-if-metageneration-match', re.I) | |
| 61 CUSTOM_GOOG_METADATA_REGEX = re.compile(r'^x-goog-meta-(?P<header_key>.*)', | |
| 62 re.I) | |
| 63 CUSTOM_AMZ_METADATA_REGEX = re.compile(r'^x-amz-meta-(?P<header_key>.*)', re.I) | |
| 64 CUSTOM_AMZ_HEADER_REGEX = re.compile(r'^x-amz-(?P<header_key>.*)', re.I) | |
| 65 | |
| 66 # gsutil-specific GUIDs for marking special metadata for S3 compatibility. | |
| 67 S3_ACL_MARKER_GUID = '3b89a6b5-b55a-4900-8c44-0b0a2f5eab43-s3-AclMarker' | |
| 68 S3_DELETE_MARKER_GUID = 'eadeeee8-fa8c-49bb-8a7d-0362215932d8-s3-DeleteMarker' | |
| 69 S3_MARKER_GUIDS = [S3_ACL_MARKER_GUID, S3_DELETE_MARKER_GUID] | |
| 70 # This distinguishes S3 custom headers from S3 metadata on objects. | |
| 71 S3_HEADER_PREFIX = 'custom-amz-header' | |
| 72 | |
| 73 DEFAULT_CONTENT_TYPE = 'application/octet-stream' | |
| 74 | |
| 75 # Because CORS is just a list in apitools, we need special handling or blank | |
| 76 # CORS lists will get sent with other configuration commands such as lifecycle, | |
| 77 # commands, which would cause CORS configuration to be unintentionally removed. | |
| 78 # Protorpc defaults list values to an empty list, and won't allow us to set the | |
| 79 # value to None like other configuration fields, so there is no way to | |
| 80 # distinguish the default value from when we actually want to remove the CORS | |
| 81 # configuration. To work around this, we create a dummy CORS entry that | |
| 82 # signifies that we should nullify the CORS configuration. | |
| 83 # A value of [] means don't modify the CORS configuration. | |
| 84 # A value of REMOVE_CORS_CONFIG means remove the CORS configuration. | |
| 85 REMOVE_CORS_CONFIG = [apitools_messages.Bucket.CorsValueListEntry( | |
| 86 maxAgeSeconds=-1, method=['REMOVE_CORS_CONFIG'])] | |
| 87 | |
| 88 | |
| 89 def ObjectMetadataFromHeaders(headers): | |
| 90 """Creates object metadata according to the provided headers. | |
| 91 | |
| 92 gsutil -h allows specifiying various headers (originally intended | |
| 93 to be passed to boto in gsutil v3). For the JSON API to be compatible with | |
| 94 this option, we need to parse these headers into gsutil_api Object fields. | |
| 95 | |
| 96 Args: | |
| 97 headers: Dict of headers passed via gsutil -h | |
| 98 | |
| 99 Raises: | |
| 100 ArgumentException if an invalid header is encountered. | |
| 101 | |
| 102 Returns: | |
| 103 apitools Object with relevant fields populated from headers. | |
| 104 """ | |
| 105 obj_metadata = apitools_messages.Object() | |
| 106 for header, value in headers.items(): | |
| 107 if CACHE_CONTROL_REGEX.match(header): | |
| 108 obj_metadata.cacheControl = value.strip() | |
| 109 elif CONTENT_DISPOSITION_REGEX.match(header): | |
| 110 obj_metadata.contentDisposition = value.strip() | |
| 111 elif CONTENT_ENCODING_REGEX.match(header): | |
| 112 obj_metadata.contentEncoding = value.strip() | |
| 113 elif CONTENT_MD5_REGEX.match(header): | |
| 114 obj_metadata.md5Hash = value.strip() | |
| 115 elif CONTENT_LANGUAGE_REGEX.match(header): | |
| 116 obj_metadata.contentLanguage = value.strip() | |
| 117 elif CONTENT_TYPE_REGEX.match(header): | |
| 118 if not value: | |
| 119 obj_metadata.contentType = DEFAULT_CONTENT_TYPE | |
| 120 else: | |
| 121 obj_metadata.contentType = value.strip() | |
| 122 elif GOOG_API_VERSION_REGEX.match(header): | |
| 123 # API version is only relevant for XML, ignore and rely on the XML API | |
| 124 # to add the appropriate version. | |
| 125 continue | |
| 126 elif GOOG_GENERATION_MATCH_REGEX.match(header): | |
| 127 # Preconditions are handled elsewhere, but allow these headers through. | |
| 128 continue | |
| 129 elif GOOG_METAGENERATION_MATCH_REGEX.match(header): | |
| 130 # Preconditions are handled elsewhere, but allow these headers through. | |
| 131 continue | |
| 132 else: | |
| 133 custom_goog_metadata_match = CUSTOM_GOOG_METADATA_REGEX.match(header) | |
| 134 custom_amz_metadata_match = CUSTOM_AMZ_METADATA_REGEX.match(header) | |
| 135 custom_amz_header_match = CUSTOM_AMZ_HEADER_REGEX.match(header) | |
| 136 header_key = None | |
| 137 if custom_goog_metadata_match: | |
| 138 header_key = custom_goog_metadata_match.group('header_key') | |
| 139 elif custom_amz_metadata_match: | |
| 140 header_key = custom_amz_metadata_match.group('header_key') | |
| 141 elif custom_amz_header_match: | |
| 142 # If we got here we are guaranteed by the prior statement that this is | |
| 143 # not an x-amz-meta- header. | |
| 144 header_key = (S3_HEADER_PREFIX + | |
| 145 custom_amz_header_match.group('header_key')) | |
| 146 if header_key: | |
| 147 if header_key.lower() == 'x-goog-content-language': | |
| 148 # Work around content-language being inserted into custom metadata. | |
| 149 continue | |
| 150 if not obj_metadata.metadata: | |
| 151 obj_metadata.metadata = apitools_messages.Object.MetadataValue() | |
| 152 if not obj_metadata.metadata.additionalProperties: | |
| 153 obj_metadata.metadata.additionalProperties = [] | |
| 154 obj_metadata.metadata.additionalProperties.append( | |
| 155 apitools_messages.Object.MetadataValue.AdditionalProperty( | |
| 156 key=header_key, value=value)) | |
| 157 else: | |
| 158 raise ArgumentException( | |
| 159 'Invalid header specifed: %s:%s' % (header, value)) | |
| 160 return obj_metadata | |
| 161 | |
| 162 | |
| 163 def HeadersFromObjectMetadata(dst_obj_metadata, provider): | |
| 164 """Creates a header dictionary based on existing object metadata. | |
| 165 | |
| 166 Args: | |
| 167 dst_obj_metadata: Object metadata to create the headers from. | |
| 168 provider: Provider string ('gs' or 's3') | |
| 169 | |
| 170 Returns: | |
| 171 Headers dictionary. | |
| 172 """ | |
| 173 headers = {} | |
| 174 if not dst_obj_metadata: | |
| 175 return | |
| 176 # Metadata values of '' mean suppress/remove this header. | |
| 177 if dst_obj_metadata.cacheControl is not None: | |
| 178 if not dst_obj_metadata.cacheControl: | |
| 179 headers['cache-control'] = None | |
| 180 else: | |
| 181 headers['cache-control'] = dst_obj_metadata.cacheControl.strip() | |
| 182 if dst_obj_metadata.contentDisposition: | |
| 183 if not dst_obj_metadata.contentDisposition: | |
| 184 headers['content-disposition'] = None | |
| 185 else: | |
| 186 headers['content-disposition'] = ( | |
| 187 dst_obj_metadata.contentDisposition.strip()) | |
| 188 if dst_obj_metadata.contentEncoding: | |
| 189 if not dst_obj_metadata.contentEncoding: | |
| 190 headers['content-encoding'] = None | |
| 191 else: | |
| 192 headers['content-encoding'] = dst_obj_metadata.contentEncoding.strip() | |
| 193 if dst_obj_metadata.contentLanguage: | |
| 194 if not dst_obj_metadata.contentLanguage: | |
| 195 headers['content-language'] = None | |
| 196 else: | |
| 197 headers['content-language'] = dst_obj_metadata.contentLanguage.strip() | |
| 198 if dst_obj_metadata.md5Hash: | |
| 199 if not dst_obj_metadata.md5Hash: | |
| 200 headers['Content-MD5'] = None | |
| 201 else: | |
| 202 headers['Content-MD5'] = dst_obj_metadata.md5Hash.strip() | |
| 203 if dst_obj_metadata.contentType is not None: | |
| 204 if not dst_obj_metadata.contentType: | |
| 205 headers['content-type'] = None | |
| 206 else: | |
| 207 headers['content-type'] = dst_obj_metadata.contentType.strip() | |
| 208 if (dst_obj_metadata.metadata and | |
| 209 dst_obj_metadata.metadata.additionalProperties): | |
| 210 for additional_property in dst_obj_metadata.metadata.additionalProperties: | |
| 211 # Work around content-language being inserted into custom metadata by | |
| 212 # the XML API. | |
| 213 if additional_property.key == 'content-language': | |
| 214 continue | |
| 215 # Don't translate special metadata markers. | |
| 216 if additional_property.key in S3_MARKER_GUIDS: | |
| 217 continue | |
| 218 if provider == 'gs': | |
| 219 header_name = 'x-goog-meta-' + additional_property.key | |
| 220 elif provider == 's3': | |
| 221 if additional_property.key.startswith(S3_HEADER_PREFIX): | |
| 222 header_name = ('x-amz-' + | |
| 223 additional_property.key[len(S3_HEADER_PREFIX):]) | |
| 224 else: | |
| 225 header_name = 'x-amz-meta-' + additional_property.key | |
| 226 else: | |
| 227 raise ArgumentException('Invalid provider specified: %s' % provider) | |
| 228 if (additional_property.value is not None and | |
| 229 not additional_property.value): | |
| 230 headers[header_name] = None | |
| 231 else: | |
| 232 headers[header_name] = additional_property.value | |
| 233 return headers | |
| 234 | |
| 235 | |
| 236 def CopyObjectMetadata(src_obj_metadata, dst_obj_metadata, override=False): | |
| 237 """Copies metadata from src_obj_metadata to dst_obj_metadata. | |
| 238 | |
| 239 Args: | |
| 240 src_obj_metadata: Metadata from source object | |
| 241 dst_obj_metadata: Initialized metadata for destination object | |
| 242 override: If true, will overwrite metadata in destination object. | |
| 243 If false, only writes metadata for values that don't already | |
| 244 exist. | |
| 245 """ | |
| 246 if override or not dst_obj_metadata.cacheControl: | |
| 247 dst_obj_metadata.cacheControl = src_obj_metadata.cacheControl | |
| 248 if override or not dst_obj_metadata.contentDisposition: | |
| 249 dst_obj_metadata.contentDisposition = src_obj_metadata.contentDisposition | |
| 250 if override or not dst_obj_metadata.contentEncoding: | |
| 251 dst_obj_metadata.contentEncoding = src_obj_metadata.contentEncoding | |
| 252 if override or not dst_obj_metadata.contentLanguage: | |
| 253 dst_obj_metadata.contentLanguage = src_obj_metadata.contentLanguage | |
| 254 if override or not dst_obj_metadata.contentType: | |
| 255 dst_obj_metadata.contentType = src_obj_metadata.contentType | |
| 256 if override or not dst_obj_metadata.md5Hash: | |
| 257 dst_obj_metadata.md5Hash = src_obj_metadata.md5Hash | |
| 258 | |
| 259 # TODO: Apitools should ideally treat metadata like a real dictionary instead | |
| 260 # of a list of key/value pairs (with an O(N^2) lookup). In practice the | |
| 261 # number of values is typically small enough not to matter. | |
| 262 # Work around this by creating our own dictionary. | |
| 263 if (src_obj_metadata.metadata and | |
| 264 src_obj_metadata.metadata.additionalProperties): | |
| 265 if not dst_obj_metadata.metadata: | |
| 266 dst_obj_metadata.metadata = apitools_messages.Object.MetadataValue() | |
| 267 if not dst_obj_metadata.metadata.additionalProperties: | |
| 268 dst_obj_metadata.metadata.additionalProperties = [] | |
| 269 dst_metadata_dict = {} | |
| 270 for dst_prop in dst_obj_metadata.metadata.additionalProperties: | |
| 271 dst_metadata_dict[dst_prop.key] = dst_prop.value | |
| 272 for src_prop in src_obj_metadata.metadata.additionalProperties: | |
| 273 if src_prop.key in dst_metadata_dict: | |
| 274 if override: | |
| 275 # Metadata values of '' mean suppress/remove this header. | |
| 276 if src_prop.value is not None and not src_prop.value: | |
| 277 dst_metadata_dict[src_prop.key] = None | |
| 278 else: | |
| 279 dst_metadata_dict[src_prop.key] = src_prop.value | |
| 280 else: | |
| 281 dst_metadata_dict[src_prop.key] = src_prop.value | |
| 282 # Rewrite the list with our updated dict. | |
| 283 dst_obj_metadata.metadata.additionalProperties = [] | |
| 284 for k, v in dst_metadata_dict.iteritems(): | |
| 285 dst_obj_metadata.metadata.additionalProperties.append( | |
| 286 apitools_messages.Object.MetadataValue.AdditionalProperty(key=k, | |
| 287 value=v)) | |
| 288 | |
| 289 | |
| 290 def PreconditionsFromHeaders(headers): | |
| 291 """Creates bucket or object preconditions acccording to the provided headers. | |
| 292 | |
| 293 Args: | |
| 294 headers: Dict of headers passed via gsutil -h | |
| 295 | |
| 296 Returns: | |
| 297 gsutil Cloud API Preconditions object fields populated from headers, or None | |
| 298 if no precondition headers are present. | |
| 299 """ | |
| 300 return_preconditions = Preconditions() | |
| 301 try: | |
| 302 for header, value in headers.items(): | |
| 303 if GOOG_GENERATION_MATCH_REGEX.match(header): | |
| 304 return_preconditions.gen_match = long(value) | |
| 305 if GOOG_METAGENERATION_MATCH_REGEX.match(header): | |
| 306 return_preconditions.meta_gen_match = long(value) | |
| 307 except ValueError, _: | |
| 308 raise ArgumentException('Invalid precondition header specified. ' | |
| 309 'x-goog-if-generation-match and ' | |
| 310 'x-goog-if-metageneration match must be specified ' | |
| 311 'with a positive integer value.') | |
| 312 return return_preconditions | |
| 313 | |
| 314 | |
| 315 def CreateBucketNotFoundException(code, provider, bucket_name): | |
| 316 return NotFoundException('%s://%s bucket does not exist.' % | |
| 317 (provider, bucket_name), status=code) | |
| 318 | |
| 319 | |
| 320 def CreateObjectNotFoundException(code, provider, bucket_name, object_name, | |
| 321 generation=None): | |
| 322 uri_string = '%s://%s/%s' % (provider, bucket_name, object_name) | |
| 323 if generation: | |
| 324 uri_string += '#%s' % str(generation) | |
| 325 return NotFoundException('%s does not exist.' % uri_string, status=code) | |
| 326 | |
| 327 | |
| 328 def EncodeStringAsLong(string_to_convert): | |
| 329 """Encodes an ASCII string as a python long. | |
| 330 | |
| 331 This is used for modeling S3 version_id's as apitools generation. Because | |
| 332 python longs can be arbitrarily large, this works. | |
| 333 | |
| 334 Args: | |
| 335 string_to_convert: ASCII string to convert to a long. | |
| 336 | |
| 337 Returns: | |
| 338 Long that represents the input string. | |
| 339 """ | |
| 340 return long(string_to_convert.encode('hex'), 16) | |
| 341 | |
| 342 | |
| 343 def _DecodeLongAsString(long_to_convert): | |
| 344 """Decodes an encoded python long into an ASCII string. | |
| 345 | |
| 346 This is used for modeling S3 version_id's as apitools generation. | |
| 347 | |
| 348 Args: | |
| 349 long_to_convert: long to convert to ASCII string. If this is already a | |
| 350 string, it is simply returned. | |
| 351 | |
| 352 Returns: | |
| 353 String decoded from the input long. | |
| 354 """ | |
| 355 if isinstance(long_to_convert, basestring): | |
| 356 # Already converted. | |
| 357 return long_to_convert | |
| 358 return hex(long_to_convert)[2:-1].decode('hex') | |
| 359 | |
| 360 | |
| 361 def GenerationFromUrlAndString(url, generation): | |
| 362 """Decodes a generation from a StorageURL and a generation string. | |
| 363 | |
| 364 This is used to represent gs and s3 versioning. | |
| 365 | |
| 366 Args: | |
| 367 url: StorageUrl representing the object. | |
| 368 generation: Long or string representing the object's generation or | |
| 369 version. | |
| 370 | |
| 371 Returns: | |
| 372 Valid generation string for use in URLs. | |
| 373 """ | |
| 374 if url.scheme == 's3' and generation: | |
| 375 return _DecodeLongAsString(generation) | |
| 376 return generation | |
| 377 | |
| 378 | |
| 379 def CheckForXmlConfigurationAndRaise(config_type_string, json_txt): | |
| 380 """Checks a JSON parse exception for provided XML configuration.""" | |
| 381 try: | |
| 382 xml.etree.ElementTree.fromstring(str(json_txt)) | |
| 383 raise ArgumentException('\n'.join(textwrap.wrap( | |
| 384 'XML {0} data provided; Google Cloud Storage {0} configuration ' | |
| 385 'now uses JSON format. To convert your {0}, set the desired XML ' | |
| 386 'ACL using \'gsutil {1} set ...\' with gsutil version 3.x. Then ' | |
| 387 'use \'gsutil {1} get ...\' with gsutil version 4 or greater to ' | |
| 388 'get the corresponding JSON {0}.'.format(config_type_string, | |
| 389 config_type_string.lower())))) | |
| 390 except XmlParseError: | |
| 391 pass | |
| 392 raise ArgumentException('JSON %s data could not be loaded ' | |
| 393 'from: %s' % (config_type_string, json_txt)) | |
| 394 | |
| 395 | |
| 396 class LifecycleTranslation(object): | |
| 397 """Functions for converting between various lifecycle formats. | |
| 398 | |
| 399 This class handles conversation to and from Boto Cors objects, JSON text, | |
| 400 and apitools Message objects. | |
| 401 """ | |
| 402 | |
| 403 @classmethod | |
| 404 def BotoLifecycleFromMessage(cls, lifecycle_message): | |
| 405 """Translates an apitools message to a boto lifecycle object.""" | |
| 406 boto_lifecycle = boto.gs.lifecycle.LifecycleConfig() | |
| 407 if lifecycle_message: | |
| 408 for rule_message in lifecycle_message.rule: | |
| 409 boto_rule = boto.gs.lifecycle.Rule() | |
| 410 if (rule_message.action and rule_message.action.type and | |
| 411 rule_message.action.type.lower() == 'delete'): | |
| 412 boto_rule.action = boto.gs.lifecycle.DELETE | |
| 413 if rule_message.condition: | |
| 414 if rule_message.condition.age: | |
| 415 boto_rule.conditions[boto.gs.lifecycle.AGE] = ( | |
| 416 str(rule_message.condition.age)) | |
| 417 if rule_message.condition.createdBefore: | |
| 418 boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE] = ( | |
| 419 str(rule_message.condition.createdBefore)) | |
| 420 if rule_message.condition.isLive: | |
| 421 boto_rule.conditions[boto.gs.lifecycle.IS_LIVE] = ( | |
| 422 str(rule_message.condition.isLive)) | |
| 423 if rule_message.condition.numNewerVersions: | |
| 424 boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS] = ( | |
| 425 str(rule_message.condition.numNewerVersions)) | |
| 426 boto_lifecycle.append(boto_rule) | |
| 427 return boto_lifecycle | |
| 428 | |
| 429 @classmethod | |
| 430 def BotoLifecycleToMessage(cls, boto_lifecycle): | |
| 431 """Translates a boto lifecycle object to an apitools message.""" | |
| 432 lifecycle_message = None | |
| 433 if boto_lifecycle: | |
| 434 lifecycle_message = apitools_messages.Bucket.LifecycleValue() | |
| 435 for boto_rule in boto_lifecycle: | |
| 436 lifecycle_rule = ( | |
| 437 apitools_messages.Bucket.LifecycleValue.RuleValueListEntry()) | |
| 438 lifecycle_rule.condition = (apitools_messages.Bucket.LifecycleValue. | |
| 439 RuleValueListEntry.ConditionValue()) | |
| 440 if boto_rule.action and boto_rule.action == boto.gs.lifecycle.DELETE: | |
| 441 lifecycle_rule.action = (apitools_messages.Bucket.LifecycleValue. | |
| 442 RuleValueListEntry.ActionValue( | |
| 443 type='Delete')) | |
| 444 if boto.gs.lifecycle.AGE in boto_rule.conditions: | |
| 445 lifecycle_rule.condition.age = int( | |
| 446 boto_rule.conditions[boto.gs.lifecycle.AGE]) | |
| 447 if boto.gs.lifecycle.CREATED_BEFORE in boto_rule.conditions: | |
| 448 lifecycle_rule.condition.createdBefore = ( | |
| 449 LifecycleTranslation.TranslateBotoLifecycleTimestamp( | |
| 450 boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE])) | |
| 451 if boto.gs.lifecycle.IS_LIVE in boto_rule.conditions: | |
| 452 lifecycle_rule.condition.isLive = bool( | |
| 453 boto_rule.conditions[boto.gs.lifecycle.IS_LIVE]) | |
| 454 if boto.gs.lifecycle.NUM_NEWER_VERSIONS in boto_rule.conditions: | |
| 455 lifecycle_rule.condition.numNewerVersions = int( | |
| 456 boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS]) | |
| 457 lifecycle_message.rule.append(lifecycle_rule) | |
| 458 return lifecycle_message | |
| 459 | |
| 460 @classmethod | |
| 461 def JsonLifecycleFromMessage(cls, lifecycle_message): | |
| 462 """Translates an apitools message to lifecycle JSON.""" | |
| 463 return str(encoding.MessageToJson(lifecycle_message)) + '\n' | |
| 464 | |
| 465 @classmethod | |
| 466 def JsonLifecycleToMessage(cls, json_txt): | |
| 467 """Translates lifecycle JSON to an apitools message.""" | |
| 468 try: | |
| 469 deserialized_lifecycle = json.loads(json_txt) | |
| 470 # If lifecycle JSON is the in the following format | |
| 471 # {'lifecycle': {'rule': ... then strip out the 'lifecycle' key | |
| 472 # and reduce it to the following format | |
| 473 # {'rule': ... | |
| 474 if 'lifecycle' in deserialized_lifecycle: | |
| 475 deserialized_lifecycle = deserialized_lifecycle['lifecycle'] | |
| 476 lifecycle = encoding.DictToMessage( | |
| 477 deserialized_lifecycle, apitools_messages.Bucket.LifecycleValue) | |
| 478 return lifecycle | |
| 479 except ValueError: | |
| 480 CheckForXmlConfigurationAndRaise('lifecycle', json_txt) | |
| 481 | |
| 482 @classmethod | |
| 483 def TranslateBotoLifecycleTimestamp(cls, lifecycle_datetime): | |
| 484 """Parses the timestamp from the boto lifecycle into a datetime object.""" | |
| 485 return datetime.datetime.strptime(lifecycle_datetime, '%Y-%m-%d').date() | |
| 486 | |
| 487 | |
| 488 class CorsTranslation(object): | |
| 489 """Functions for converting between various CORS formats. | |
| 490 | |
| 491 This class handles conversation to and from Boto Cors objects, JSON text, | |
| 492 and apitools Message objects. | |
| 493 """ | |
| 494 | |
| 495 @classmethod | |
| 496 def BotoCorsFromMessage(cls, cors_message): | |
| 497 """Translates an apitools message to a boto Cors object.""" | |
| 498 cors = boto.gs.cors.Cors() | |
| 499 cors.cors = [] | |
| 500 for collection_message in cors_message: | |
| 501 collection_elements = [] | |
| 502 if collection_message.maxAgeSeconds: | |
| 503 collection_elements.append((boto.gs.cors.MAXAGESEC, | |
| 504 str(collection_message.maxAgeSeconds))) | |
| 505 if collection_message.method: | |
| 506 method_elements = [] | |
| 507 for method in collection_message.method: | |
| 508 method_elements.append((boto.gs.cors.METHOD, method)) | |
| 509 collection_elements.append((boto.gs.cors.METHODS, method_elements)) | |
| 510 if collection_message.origin: | |
| 511 origin_elements = [] | |
| 512 for origin in collection_message.origin: | |
| 513 origin_elements.append((boto.gs.cors.ORIGIN, origin)) | |
| 514 collection_elements.append((boto.gs.cors.ORIGINS, origin_elements)) | |
| 515 if collection_message.responseHeader: | |
| 516 header_elements = [] | |
| 517 for header in collection_message.responseHeader: | |
| 518 header_elements.append((boto.gs.cors.HEADER, header)) | |
| 519 collection_elements.append((boto.gs.cors.HEADERS, header_elements)) | |
| 520 cors.cors.append(collection_elements) | |
| 521 return cors | |
| 522 | |
| 523 @classmethod | |
| 524 def BotoCorsToMessage(cls, boto_cors): | |
| 525 """Translates a boto Cors object to an apitools message.""" | |
| 526 message_cors = [] | |
| 527 if boto_cors.cors: | |
| 528 for cors_collection in boto_cors.cors: | |
| 529 if cors_collection: | |
| 530 collection_message = apitools_messages.Bucket.CorsValueListEntry() | |
| 531 for element_tuple in cors_collection: | |
| 532 if element_tuple[0] == boto.gs.cors.MAXAGESEC: | |
| 533 collection_message.maxAgeSeconds = int(element_tuple[1]) | |
| 534 if element_tuple[0] == boto.gs.cors.METHODS: | |
| 535 for method_tuple in element_tuple[1]: | |
| 536 collection_message.method.append(method_tuple[1]) | |
| 537 if element_tuple[0] == boto.gs.cors.ORIGINS: | |
| 538 for origin_tuple in element_tuple[1]: | |
| 539 collection_message.origin.append(origin_tuple[1]) | |
| 540 if element_tuple[0] == boto.gs.cors.HEADERS: | |
| 541 for header_tuple in element_tuple[1]: | |
| 542 collection_message.responseHeader.append(header_tuple[1]) | |
| 543 message_cors.append(collection_message) | |
| 544 return message_cors | |
| 545 | |
| 546 @classmethod | |
| 547 def JsonCorsToMessageEntries(cls, json_cors): | |
| 548 """Translates CORS JSON to an apitools message. | |
| 549 | |
| 550 Args: | |
| 551 json_cors: JSON string representing CORS configuration. | |
| 552 | |
| 553 Returns: | |
| 554 List of apitools Bucket.CorsValueListEntry. An empty list represents | |
| 555 no CORS configuration. | |
| 556 """ | |
| 557 try: | |
| 558 deserialized_cors = json.loads(json_cors) | |
| 559 cors = [] | |
| 560 for cors_entry in deserialized_cors: | |
| 561 cors.append(encoding.DictToMessage( | |
| 562 cors_entry, apitools_messages.Bucket.CorsValueListEntry)) | |
| 563 return cors | |
| 564 except ValueError: | |
| 565 CheckForXmlConfigurationAndRaise('CORS', json_cors) | |
| 566 | |
| 567 @classmethod | |
| 568 def MessageEntriesToJson(cls, cors_message): | |
| 569 """Translates an apitools message to CORS JSON.""" | |
| 570 json_text = '' | |
| 571 # Because CORS is a MessageField, serialize/deserialize as JSON list. | |
| 572 json_text += '[' | |
| 573 printed_one = False | |
| 574 for cors_entry in cors_message: | |
| 575 if printed_one: | |
| 576 json_text += ',' | |
| 577 else: | |
| 578 printed_one = True | |
| 579 json_text += encoding.MessageToJson(cors_entry) | |
| 580 json_text += ']\n' | |
| 581 return json_text | |
| 582 | |
| 583 | |
| 584 def S3MarkerAclFromObjectMetadata(object_metadata): | |
| 585 """Retrieves GUID-marked S3 ACL from object metadata, if present. | |
| 586 | |
| 587 Args: | |
| 588 object_metadata: Object metadata to check. | |
| 589 | |
| 590 Returns: | |
| 591 S3 ACL text, if present, None otherwise. | |
| 592 """ | |
| 593 if (object_metadata and object_metadata.metadata and | |
| 594 object_metadata.metadata.additionalProperties): | |
| 595 for prop in object_metadata.metadata.additionalProperties: | |
| 596 if prop.key == S3_ACL_MARKER_GUID: | |
| 597 return prop.value | |
| 598 | |
| 599 | |
| 600 def AddS3MarkerAclToObjectMetadata(object_metadata, acl_text): | |
| 601 """Adds a GUID-marked S3 ACL to the object metadata. | |
| 602 | |
| 603 Args: | |
| 604 object_metadata: Object metadata to add the acl to. | |
| 605 acl_text: S3 ACL text to add. | |
| 606 """ | |
| 607 if not object_metadata.metadata: | |
| 608 object_metadata.metadata = apitools_messages.Object.MetadataValue() | |
| 609 if not object_metadata.metadata.additionalProperties: | |
| 610 object_metadata.metadata.additionalProperties = [] | |
| 611 | |
| 612 object_metadata.metadata.additionalProperties.append( | |
| 613 apitools_messages.Object.MetadataValue.AdditionalProperty( | |
| 614 key=S3_ACL_MARKER_GUID, value=acl_text)) | |
| 615 | |
| 616 | |
| 617 class AclTranslation(object): | |
| 618 """Functions for converting between various ACL formats. | |
| 619 | |
| 620 This class handles conversion to and from Boto ACL objects, JSON text, | |
| 621 and apitools Message objects. | |
| 622 """ | |
| 623 | |
| 624 JSON_TO_XML_ROLES = {'READER': 'READ', 'WRITER': 'WRITE', | |
| 625 'OWNER': 'FULL_CONTROL'} | |
| 626 XML_TO_JSON_ROLES = {'READ': 'READER', 'WRITE': 'WRITER', | |
| 627 'FULL_CONTROL': 'OWNER'} | |
| 628 | |
| 629 @classmethod | |
| 630 def BotoAclFromJson(cls, acl_json): | |
| 631 acl = ACL() | |
| 632 acl.parent = None | |
| 633 acl.entries = cls.BotoEntriesFromJson(acl_json, acl) | |
| 634 return acl | |
| 635 | |
| 636 @classmethod | |
| 637 # acl_message is a list of messages, either object or bucketaccesscontrol | |
| 638 def BotoAclFromMessage(cls, acl_message): | |
| 639 acl_dicts = [] | |
| 640 for message in acl_message: | |
| 641 acl_dicts.append(encoding.MessageToDict(message)) | |
| 642 return cls.BotoAclFromJson(acl_dicts) | |
| 643 | |
| 644 @classmethod | |
| 645 def BotoAclToJson(cls, acl): | |
| 646 if hasattr(acl, 'entries'): | |
| 647 return cls.BotoEntriesToJson(acl.entries) | |
| 648 return [] | |
| 649 | |
| 650 @classmethod | |
| 651 def BotoObjectAclToMessage(cls, acl): | |
| 652 for entry in cls.BotoAclToJson(acl): | |
| 653 message = encoding.DictToMessage(entry, | |
| 654 apitools_messages.ObjectAccessControl) | |
| 655 message.kind = u'storage#objectAccessControl' | |
| 656 yield message | |
| 657 | |
| 658 @classmethod | |
| 659 def BotoBucketAclToMessage(cls, acl): | |
| 660 for entry in cls.BotoAclToJson(acl): | |
| 661 message = encoding.DictToMessage(entry, | |
| 662 apitools_messages.BucketAccessControl) | |
| 663 message.kind = u'storage#bucketAccessControl' | |
| 664 yield message | |
| 665 | |
| 666 @classmethod | |
| 667 def BotoEntriesFromJson(cls, acl_json, parent): | |
| 668 entries = Entries(parent) | |
| 669 entries.parent = parent | |
| 670 entries.entry_list = [cls.BotoEntryFromJson(entry_json) | |
| 671 for entry_json in acl_json] | |
| 672 return entries | |
| 673 | |
| 674 @classmethod | |
| 675 def BotoEntriesToJson(cls, entries): | |
| 676 return [cls.BotoEntryToJson(entry) for entry in entries.entry_list] | |
| 677 | |
| 678 @classmethod | |
| 679 def BotoEntryFromJson(cls, entry_json): | |
| 680 """Converts a JSON entry into a Boto ACL entry.""" | |
| 681 entity = entry_json['entity'] | |
| 682 permission = cls.JSON_TO_XML_ROLES[entry_json['role']] | |
| 683 if entity.lower() == ALL_USERS.lower(): | |
| 684 return Entry(type=ALL_USERS, permission=permission) | |
| 685 elif entity.lower() == ALL_AUTHENTICATED_USERS.lower(): | |
| 686 return Entry(type=ALL_AUTHENTICATED_USERS, permission=permission) | |
| 687 elif entity.startswith('project'): | |
| 688 raise CommandException('XML API does not support project scopes, ' | |
| 689 'cannot translate ACL.') | |
| 690 elif 'email' in entry_json: | |
| 691 if entity.startswith('user'): | |
| 692 scope_type = USER_BY_EMAIL | |
| 693 elif entity.startswith('group'): | |
| 694 scope_type = GROUP_BY_EMAIL | |
| 695 return Entry(type=scope_type, email_address=entry_json['email'], | |
| 696 permission=permission) | |
| 697 elif 'entityId' in entry_json: | |
| 698 if entity.startswith('user'): | |
| 699 scope_type = USER_BY_ID | |
| 700 elif entity.startswith('group'): | |
| 701 scope_type = GROUP_BY_ID | |
| 702 return Entry(type=scope_type, id=entry_json['entityId'], | |
| 703 permission=permission) | |
| 704 elif 'domain' in entry_json: | |
| 705 if entity.startswith('domain'): | |
| 706 scope_type = GROUP_BY_DOMAIN | |
| 707 return Entry(type=scope_type, domain=entry_json['domain'], | |
| 708 permission=permission) | |
| 709 raise CommandException('Failed to translate JSON ACL to XML.') | |
| 710 | |
| 711 @classmethod | |
| 712 def BotoEntryToJson(cls, entry): | |
| 713 """Converts a Boto ACL entry to a valid JSON dictionary.""" | |
| 714 acl_entry_json = {} | |
| 715 # JSON API documentation uses camel case. | |
| 716 scope_type_lower = entry.scope.type.lower() | |
| 717 if scope_type_lower == ALL_USERS.lower(): | |
| 718 acl_entry_json['entity'] = 'allUsers' | |
| 719 elif scope_type_lower == ALL_AUTHENTICATED_USERS.lower(): | |
| 720 acl_entry_json['entity'] = 'allAuthenticatedUsers' | |
| 721 elif scope_type_lower == USER_BY_EMAIL.lower(): | |
| 722 acl_entry_json['entity'] = 'user-%s' % entry.scope.email_address | |
| 723 acl_entry_json['email'] = entry.scope.email_address | |
| 724 elif scope_type_lower == USER_BY_ID.lower(): | |
| 725 acl_entry_json['entity'] = 'user-%s' % entry.scope.id | |
| 726 acl_entry_json['entityId'] = entry.scope.id | |
| 727 elif scope_type_lower == GROUP_BY_EMAIL.lower(): | |
| 728 acl_entry_json['entity'] = 'group-%s' % entry.scope.email_address | |
| 729 acl_entry_json['email'] = entry.scope.email_address | |
| 730 elif scope_type_lower == GROUP_BY_ID.lower(): | |
| 731 acl_entry_json['entity'] = 'group-%s' % entry.scope.id | |
| 732 acl_entry_json['entityId'] = entry.scope.id | |
| 733 elif scope_type_lower == GROUP_BY_DOMAIN.lower(): | |
| 734 acl_entry_json['entity'] = 'domain-%s' % entry.scope.domain | |
| 735 acl_entry_json['domain'] = entry.scope.domain | |
| 736 else: | |
| 737 raise ArgumentException('ACL contains invalid scope type: %s' % | |
| 738 scope_type_lower) | |
| 739 | |
| 740 acl_entry_json['role'] = cls.XML_TO_JSON_ROLES[entry.permission] | |
| 741 return acl_entry_json | |
| 742 | |
| 743 @classmethod | |
| 744 def JsonToMessage(cls, json_data, message_type): | |
| 745 """Converts the input JSON data into list of Object/BucketAccessControls. | |
| 746 | |
| 747 Args: | |
| 748 json_data: String of JSON to convert. | |
| 749 message_type: Which type of access control entries to return, | |
| 750 either ObjectAccessControl or BucketAccessControl. | |
| 751 | |
| 752 Raises: | |
| 753 ArgumentException on invalid JSON data. | |
| 754 | |
| 755 Returns: | |
| 756 List of ObjectAccessControl or BucketAccessControl elements. | |
| 757 """ | |
| 758 try: | |
| 759 deserialized_acl = json.loads(json_data) | |
| 760 | |
| 761 acl = [] | |
| 762 for acl_entry in deserialized_acl: | |
| 763 acl.append(encoding.DictToMessage(acl_entry, message_type)) | |
| 764 return acl | |
| 765 except ValueError: | |
| 766 CheckForXmlConfigurationAndRaise('ACL', json_data) | |
| 767 | |
| 768 @classmethod | |
| 769 def JsonFromMessage(cls, acl): | |
| 770 """Strips unnecessary fields from an ACL message and returns valid JSON. | |
| 771 | |
| 772 Args: | |
| 773 acl: iterable ObjectAccessControl or BucketAccessControl | |
| 774 | |
| 775 Returns: | |
| 776 ACL JSON string. | |
| 777 """ | |
| 778 serializable_acl = [] | |
| 779 if acl is not None: | |
| 780 for acl_entry in acl: | |
| 781 if acl_entry.kind == u'storage#objectAccessControl': | |
| 782 acl_entry.object = None | |
| 783 acl_entry.generation = None | |
| 784 acl_entry.kind = None | |
| 785 acl_entry.bucket = None | |
| 786 acl_entry.id = None | |
| 787 acl_entry.selfLink = None | |
| 788 acl_entry.etag = None | |
| 789 serializable_acl.append(encoding.MessageToDict(acl_entry)) | |
| 790 return json.dumps(serializable_acl, sort_keys=True, | |
| 791 indent=2, separators=(',', ': ')) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1260493004:
Revert "Add gsutil 4.13 to telemetry/third_party" (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master