| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- coding: utf-8 -*- | |
| 2 # Copyright 2013 Google Inc. All Rights Reserved. | |
| 3 # | |
| 4 # Licensed under the Apache License, Version 2.0 (the "License"); | |
| 5 # you may not use this file except in compliance with the License. | |
| 6 # You may obtain a copy of the License at | |
| 7 # | |
| 8 # http://www.apache.org/licenses/LICENSE-2.0 | |
| 9 # | |
| 10 # Unless required by applicable law or agreed to in writing, software | |
| 11 # distributed under the License is distributed on an "AS IS" BASIS, | |
| 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 13 # See the License for the specific language governing permissions and | |
| 14 # limitations under the License. | |
| 15 """Gsutil API delegator for interacting with cloud storage providers.""" | |
| 16 | |
| 17 from __future__ import absolute_import | |
| 18 | |
| 19 import boto | |
| 20 from boto import config | |
| 21 from gslib.cloud_api import ArgumentException | |
| 22 from gslib.cloud_api import CloudApi | |
| 23 from gslib.cs_api_map import ApiMapConstants | |
| 24 from gslib.cs_api_map import ApiSelector | |
| 25 | |
| 26 | |
| 27 class CloudApiDelegator(CloudApi): | |
| 28 """Class that handles delegating requests to gsutil Cloud API implementations. | |
| 29 | |
| 30 This class is responsible for determining at runtime which gsutil Cloud API | |
| 31 implementation should service the request based on the Cloud storage provider, | |
| 32 command-level API support, and configuration file override. | |
| 33 | |
| 34 During initialization it takes as an argument a gsutil_api_map which maps | |
| 35 providers to their default and supported gsutil Cloud API implementations | |
| 36 (see comments in cs_api_map for details). | |
| 37 | |
| 38 Instantiation of multiple delegators per-thread is required for multiprocess | |
| 39 and/or multithreaded operations. Calling methods on the same delegator in | |
| 40 multiple threads is unsafe. | |
| 41 """ | |
| 42 | |
| 43 def __init__(self, bucket_storage_uri_class, gsutil_api_map, logger, | |
| 44 provider=None, debug=0): | |
| 45 """Performs necessary setup for delegating cloud storage requests. | |
| 46 | |
| 47 This function has different arguments than the gsutil Cloud API __init__ | |
| 48 function because of the delegation responsibilties of this class. | |
| 49 | |
| 50 Args: | |
| 51 bucket_storage_uri_class: boto storage_uri class, used by APIs that | |
| 52 provide boto translation or mocking. | |
| 53 gsutil_api_map: Map of providers and API selector tuples to api classes | |
| 54 which can be used to communicate with those providers. | |
| 55 logger: logging.logger for outputting log messages. | |
| 56 provider: Default provider prefix describing cloud storage provider to | |
| 57 connect to. | |
| 58 debug: Debug level for the API implementation (0..3). | |
| 59 """ | |
| 60 super(CloudApiDelegator, self).__init__(bucket_storage_uri_class, logger, | |
| 61 provider=provider, debug=debug) | |
| 62 self.api_map = gsutil_api_map | |
| 63 self.prefer_api = boto.config.get('GSUtil', 'prefer_api', '').upper() | |
| 64 self.loaded_apis = {} | |
| 65 | |
| 66 if not self.api_map[ApiMapConstants.API_MAP]: | |
| 67 raise ArgumentException('No apiclass supplied for gsutil Cloud API map.') | |
| 68 | |
| 69 def _GetApi(self, provider): | |
| 70 """Returns a valid CloudApi for use by the caller. | |
| 71 | |
| 72 This function lazy-loads connection and credentials using the API map | |
| 73 and credential store provided during class initialization. | |
| 74 | |
| 75 Args: | |
| 76 provider: Provider to load API for. If None, class-wide default is used. | |
| 77 | |
| 78 Raises: | |
| 79 ArgumentException if there is no matching API available in the API map. | |
| 80 | |
| 81 Returns: | |
| 82 Valid API instance that can be used to communicate with the Cloud | |
| 83 Storage provider. | |
| 84 """ | |
| 85 provider = provider or self.provider | |
| 86 if not provider: | |
| 87 raise ArgumentException('No provider selected for _GetApi') | |
| 88 | |
| 89 provider = str(provider) | |
| 90 if provider not in self.loaded_apis: | |
| 91 self.loaded_apis[provider] = {} | |
| 92 | |
| 93 api_selector = self.GetApiSelector(provider) | |
| 94 if api_selector not in self.loaded_apis[provider]: | |
| 95 # Need to load the API. | |
| 96 self._LoadApi(provider, api_selector) | |
| 97 | |
| 98 return self.loaded_apis[provider][api_selector] | |
| 99 | |
| 100 def _LoadApi(self, provider, api_selector): | |
| 101 """Loads a CloudApi into the loaded_apis map for this class. | |
| 102 | |
| 103 Args: | |
| 104 provider: Provider to load the API for. | |
| 105 api_selector: cs_api_map.ApiSelector defining the API type. | |
| 106 """ | |
| 107 if provider not in self.api_map[ApiMapConstants.API_MAP]: | |
| 108 raise ArgumentException( | |
| 109 'gsutil Cloud API map contains no entry for provider %s.' % provider) | |
| 110 if api_selector not in self.api_map[ApiMapConstants.API_MAP][provider]: | |
| 111 raise ArgumentException( | |
| 112 'gsutil Cloud API map does not support API %s for provider %s.' % | |
| 113 (api_selector, provider)) | |
| 114 self.loaded_apis[provider][api_selector] = ( | |
| 115 self.api_map[ApiMapConstants.API_MAP][provider][api_selector]( | |
| 116 self.bucket_storage_uri_class, | |
| 117 self.logger, | |
| 118 provider=provider, | |
| 119 debug=self.debug)) | |
| 120 | |
| 121 def GetApiSelector(self, provider=None): | |
| 122 """Returns a cs_api_map.ApiSelector based on input and configuration. | |
| 123 | |
| 124 Args: | |
| 125 provider: Provider to return the ApiSelector for. If None, class-wide | |
| 126 default is used. | |
| 127 | |
| 128 Returns: | |
| 129 cs_api_map.ApiSelector that will be used for calls to the delegator | |
| 130 for this provider. | |
| 131 """ | |
| 132 selected_provider = provider or self.provider | |
| 133 if not selected_provider: | |
| 134 raise ArgumentException('No provider selected for CloudApi') | |
| 135 | |
| 136 if (selected_provider not in self.api_map[ApiMapConstants.DEFAULT_MAP] or | |
| 137 self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider] not in | |
| 138 self.api_map[ApiMapConstants.API_MAP][selected_provider]): | |
| 139 raise ArgumentException('No default api available for provider %s' % | |
| 140 selected_provider) | |
| 141 | |
| 142 if selected_provider not in self.api_map[ApiMapConstants.SUPPORT_MAP]: | |
| 143 raise ArgumentException('No supported apis available for provider %s' % | |
| 144 selected_provider) | |
| 145 | |
| 146 api = self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider] | |
| 147 | |
| 148 # If we have only HMAC credentials for Google Cloud Storage, we must use | |
| 149 # the XML API as the JSON API does not support HMAC. | |
| 150 # | |
| 151 # Technically if we have only HMAC credentials, we should still be able to | |
| 152 # access public read resources via the JSON API, but the XML API can do | |
| 153 # that just as well. It is better to use it than inspect the credentials on | |
| 154 # every HTTP call. | |
| 155 if (provider == 'gs' and | |
| 156 not config.has_option('Credentials', 'gs_oauth2_refresh_token') and | |
| 157 not (config.has_option('Credentials', 'gs_service_client_id') | |
| 158 and config.has_option('Credentials', 'gs_service_key_file')) and | |
| 159 (config.has_option('Credentials', 'gs_access_key_id') | |
| 160 and config.has_option('Credentials', 'gs_secret_access_key'))): | |
| 161 api = ApiSelector.XML | |
| 162 # Try to force the user's preference to a supported API. | |
| 163 elif self.prefer_api in (self.api_map[ApiMapConstants.SUPPORT_MAP] | |
| 164 [selected_provider]): | |
| 165 api = self.prefer_api | |
| 166 return api | |
| 167 | |
| 168 # For function docstrings, see CloudApi class. | |
| 169 def GetBucket(self, bucket_name, provider=None, fields=None): | |
| 170 return self._GetApi(provider).GetBucket(bucket_name, fields=fields) | |
| 171 | |
| 172 def ListBuckets(self, project_id=None, provider=None, fields=None): | |
| 173 return self._GetApi(provider).ListBuckets(project_id=project_id, | |
| 174 fields=fields) | |
| 175 | |
| 176 def PatchBucket(self, bucket_name, metadata, canned_acl=None, | |
| 177 canned_def_acl=None, preconditions=None, provider=None, | |
| 178 fields=None): | |
| 179 return self._GetApi(provider).PatchBucket( | |
| 180 bucket_name, metadata, canned_acl=canned_acl, | |
| 181 canned_def_acl=canned_def_acl, preconditions=preconditions, | |
| 182 fields=fields) | |
| 183 | |
| 184 def CreateBucket(self, bucket_name, project_id=None, metadata=None, | |
| 185 provider=None, fields=None): | |
| 186 return self._GetApi(provider).CreateBucket( | |
| 187 bucket_name, project_id=project_id, metadata=metadata, fields=fields) | |
| 188 | |
| 189 def DeleteBucket(self, bucket_name, preconditions=None, provider=None): | |
| 190 return self._GetApi(provider).DeleteBucket(bucket_name, | |
| 191 preconditions=preconditions) | |
| 192 | |
| 193 def ListObjects(self, bucket_name, prefix=None, delimiter=None, | |
| 194 all_versions=None, provider=None, fields=None): | |
| 195 return self._GetApi(provider).ListObjects( | |
| 196 bucket_name, prefix=prefix, delimiter=delimiter, | |
| 197 all_versions=all_versions, fields=fields) | |
| 198 | |
| 199 def GetObjectMetadata(self, bucket_name, object_name, generation=None, | |
| 200 provider=None, fields=None): | |
| 201 return self._GetApi(provider).GetObjectMetadata( | |
| 202 bucket_name, object_name, generation=generation, fields=fields) | |
| 203 | |
| 204 def PatchObjectMetadata(self, bucket_name, object_name, metadata, | |
| 205 canned_acl=None, generation=None, preconditions=None, | |
| 206 provider=None, fields=None): | |
| 207 return self._GetApi(provider).PatchObjectMetadata( | |
| 208 bucket_name, object_name, metadata, canned_acl=canned_acl, | |
| 209 generation=generation, preconditions=preconditions, fields=fields) | |
| 210 | |
| 211 def GetObjectMedia( | |
| 212 self, bucket_name, object_name, download_stream, provider=None, | |
| 213 generation=None, object_size=None, | |
| 214 download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, | |
| 215 start_byte=0, end_byte=None, progress_callback=None, | |
| 216 serialization_data=None, digesters=None): | |
| 217 return self._GetApi(provider).GetObjectMedia( | |
| 218 bucket_name, object_name, download_stream, | |
| 219 download_strategy=download_strategy, start_byte=start_byte, | |
| 220 end_byte=end_byte, generation=generation, object_size=object_size, | |
| 221 progress_callback=progress_callback, | |
| 222 serialization_data=serialization_data, digesters=digesters) | |
| 223 | |
| 224 def UploadObject(self, upload_stream, object_metadata, size=None, | |
| 225 canned_acl=None, preconditions=None, progress_callback=None, | |
| 226 provider=None, fields=None): | |
| 227 return self._GetApi(provider).UploadObject( | |
| 228 upload_stream, object_metadata, size=size, canned_acl=canned_acl, | |
| 229 preconditions=preconditions, progress_callback=progress_callback, | |
| 230 fields=fields) | |
| 231 | |
| 232 def UploadObjectStreaming(self, upload_stream, object_metadata, | |
| 233 canned_acl=None, preconditions=None, | |
| 234 progress_callback=None, provider=None, fields=None): | |
| 235 return self._GetApi(provider).UploadObjectStreaming( | |
| 236 upload_stream, object_metadata, canned_acl=canned_acl, | |
| 237 preconditions=preconditions, progress_callback=progress_callback, | |
| 238 fields=fields) | |
| 239 | |
| 240 def UploadObjectResumable( | |
| 241 self, upload_stream, object_metadata, canned_acl=None, preconditions=None, | |
| 242 provider=None, fields=None, size=None, serialization_data=None, | |
| 243 tracker_callback=None, progress_callback=None): | |
| 244 return self._GetApi(provider).UploadObjectResumable( | |
| 245 upload_stream, object_metadata, canned_acl=canned_acl, | |
| 246 preconditions=preconditions, size=size, fields=fields, | |
| 247 serialization_data=serialization_data, | |
| 248 tracker_callback=tracker_callback, progress_callback=progress_callback) | |
| 249 | |
| 250 def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, | |
| 251 canned_acl=None, preconditions=None, progress_callback=None, | |
| 252 max_bytes_per_call=None, provider=None, fields=None): | |
| 253 return self._GetApi(provider).CopyObject( | |
| 254 src_obj_metadata, dst_obj_metadata, src_generation=src_generation, | |
| 255 canned_acl=canned_acl, preconditions=preconditions, | |
| 256 progress_callback=progress_callback, | |
| 257 max_bytes_per_call=max_bytes_per_call, fields=fields) | |
| 258 | |
| 259 def ComposeObject(self, src_objs_metadata, dst_obj_metadata, | |
| 260 preconditions=None, provider=None, fields=None): | |
| 261 return self._GetApi(provider).ComposeObject( | |
| 262 src_objs_metadata, dst_obj_metadata, preconditions=preconditions, | |
| 263 fields=fields) | |
| 264 | |
| 265 def DeleteObject(self, bucket_name, object_name, preconditions=None, | |
| 266 generation=None, provider=None): | |
| 267 return self._GetApi(provider).DeleteObject( | |
| 268 bucket_name, object_name, preconditions=preconditions, | |
| 269 generation=generation) | |
| 270 | |
| 271 def WatchBucket(self, bucket_name, address, channel_id, token=None, | |
| 272 provider=None, fields=None): | |
| 273 return self._GetApi(provider).WatchBucket( | |
| 274 bucket_name, address, channel_id, token=token, fields=fields) | |
| 275 | |
| 276 def StopChannel(self, channel_id, resource_id, provider=None): | |
| 277 return self._GetApi(provider).StopChannel(channel_id, resource_id) | |
| 278 | |
| 279 def XmlPassThroughGetAcl(self, storage_url, def_obj_acl=False, provider=None): | |
| 280 """XML compatibility function for getting ACLs. | |
| 281 | |
| 282 Args: | |
| 283 storage_url: StorageUrl object. | |
| 284 def_obj_acl: If true, get the default object ACL on a bucket. | |
| 285 provider: Cloud storage provider to connect to. If not present, | |
| 286 class-wide default is used. | |
| 287 | |
| 288 Raises: | |
| 289 ArgumentException for errors during input validation. | |
| 290 ServiceException for errors interacting with cloud storage providers. | |
| 291 | |
| 292 Returns: | |
| 293 ACL XML for the resource specified by storage_url. | |
| 294 """ | |
| 295 return self._GetApi(provider).XmlPassThroughGetAcl(storage_url, | |
| 296 def_obj_acl=def_obj_acl) | |
| 297 | |
| 298 def XmlPassThroughSetAcl(self, acl_text, storage_url, canned=True, | |
| 299 def_obj_acl=False, provider=None): | |
| 300 """XML compatibility function for setting ACLs. | |
| 301 | |
| 302 Args: | |
| 303 acl_text: XML ACL or canned ACL string. | |
| 304 storage_url: StorageUrl object. | |
| 305 canned: If true, acl_text is treated as a canned ACL string. | |
| 306 def_obj_acl: If true, set the default object ACL on a bucket. | |
| 307 provider: Cloud storage provider to connect to. If not present, | |
| 308 class-wide default is used. | |
| 309 | |
| 310 Raises: | |
| 311 ArgumentException for errors during input validation. | |
| 312 ServiceException for errors interacting with cloud storage providers. | |
| 313 | |
| 314 Returns: | |
| 315 None. | |
| 316 """ | |
| 317 self._GetApi(provider).XmlPassThroughSetAcl( | |
| 318 acl_text, storage_url, canned=canned, def_obj_acl=def_obj_acl) | |
| 319 | |
| 320 def XmlPassThroughGetCors(self, storage_url, provider=None): | |
| 321 """XML compatibility function for getting CORS configuration on a bucket. | |
| 322 | |
| 323 Args: | |
| 324 storage_url: StorageUrl object. | |
| 325 provider: Cloud storage provider to connect to. If not present, | |
| 326 class-wide default is used. | |
| 327 | |
| 328 Raises: | |
| 329 ArgumentException for errors during input validation. | |
| 330 ServiceException for errors interacting with cloud storage providers. | |
| 331 | |
| 332 Returns: | |
| 333 CORS configuration XML for the bucket specified by storage_url. | |
| 334 """ | |
| 335 return self._GetApi(provider).XmlPassThroughGetCors(storage_url) | |
| 336 | |
| 337 def XmlPassThroughSetCors(self, cors_text, storage_url, provider=None): | |
| 338 """XML compatibility function for setting CORS configuration on a bucket. | |
| 339 | |
| 340 Args: | |
| 341 cors_text: Raw CORS XML string. | |
| 342 storage_url: StorageUrl object. | |
| 343 provider: Cloud storage provider to connect to. If not present, | |
| 344 class-wide default is used. | |
| 345 | |
| 346 Raises: | |
| 347 ArgumentException for errors during input validation. | |
| 348 ServiceException for errors interacting with cloud storage providers. | |
| 349 | |
| 350 Returns: | |
| 351 None. | |
| 352 """ | |
| 353 self._GetApi(provider).XmlPassThroughSetCors(cors_text, storage_url) | |
| 354 | |
| 355 def XmlPassThroughGetLifecycle(self, storage_url, provider=None): | |
| 356 """XML compatibility function for getting lifecycle config on a bucket. | |
| 357 | |
| 358 Args: | |
| 359 storage_url: StorageUrl object. | |
| 360 provider: Cloud storage provider to connect to. If not present, | |
| 361 class-wide default is used. | |
| 362 | |
| 363 Raises: | |
| 364 ArgumentException for errors during input validation. | |
| 365 ServiceException for errors interacting with cloud storage providers. | |
| 366 | |
| 367 Returns: | |
| 368 Lifecycle configuration XML for the bucket specified by storage_url. | |
| 369 """ | |
| 370 return self._GetApi(provider).XmlPassThroughGetLifecycle(storage_url) | |
| 371 | |
| 372 def XmlPassThroughSetLifecycle(self, lifecycle_text, storage_url, | |
| 373 provider=None): | |
| 374 """XML compatibility function for setting CORS configuration on a bucket. | |
| 375 | |
| 376 Args: | |
| 377 lifecycle_text: Raw lifecycle configuration XML string. | |
| 378 storage_url: StorageUrl object. | |
| 379 provider: Cloud storage provider to connect to. If not present, | |
| 380 class-wide default is used. | |
| 381 | |
| 382 Raises: | |
| 383 ArgumentException for errors during input validation. | |
| 384 ServiceException for errors interacting with cloud storage providers. | |
| 385 | |
| 386 Returns: | |
| 387 None. | |
| 388 """ | |
| 389 self._GetApi(provider).XmlPassThroughSetLifecycle(lifecycle_text, | |
| 390 storage_url) | |
| 391 | |
| 392 def XmlPassThroughGetLogging(self, storage_url, provider=None): | |
| 393 """XML compatibility function for getting logging configuration on a bucket. | |
| 394 | |
| 395 Args: | |
| 396 storage_url: StorageUrl object. | |
| 397 provider: Cloud storage provider to connect to. If not present, | |
| 398 class-wide default is used. | |
| 399 | |
| 400 Raises: | |
| 401 ArgumentException for errors during input validation. | |
| 402 ServiceException for errors interacting with cloud storage providers. | |
| 403 | |
| 404 Returns: | |
| 405 Logging configuration XML for the bucket specified by storage_url. | |
| 406 """ | |
| 407 return self._GetApi(provider).XmlPassThroughGetLogging(storage_url) | |
| 408 | |
| 409 def XmlPassThroughGetWebsite(self, storage_url, provider=None): | |
| 410 """XML compatibility function for getting website configuration on a bucket. | |
| 411 | |
| 412 Args: | |
| 413 storage_url: StorageUrl object. | |
| 414 provider: Cloud storage provider to connect to. If not present, | |
| 415 class-wide default is used. | |
| 416 | |
| 417 Raises: | |
| 418 ArgumentException for errors during input validation. | |
| 419 ServiceException for errors interacting with cloud storage providers. | |
| 420 | |
| 421 Returns: | |
| 422 Website configuration XML for the bucket specified by storage_url. | |
| 423 """ | |
| 424 return self._GetApi(provider).XmlPassThroughGetWebsite(storage_url) | |
| 425 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1260493004:
Revert "Add gsutil 4.13 to telemetry/third_party" (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master