Index: tools/telemetry/third_party/gsutil/gslib/cloud_api.py |
diff --git a/tools/telemetry/third_party/gsutil/gslib/cloud_api.py b/tools/telemetry/third_party/gsutil/gslib/cloud_api.py |
deleted file mode 100644 |
index b7af6b6eeac56ac0457c213753aa18d7878f5edb..0000000000000000000000000000000000000000 |
--- a/tools/telemetry/third_party/gsutil/gslib/cloud_api.py |
+++ /dev/null |
@@ -1,624 +0,0 @@ |
-# -*- coding: utf-8 -*- |
-# Copyright 2013 Google Inc. All Rights Reserved. |
-# |
-# Licensed under the Apache License, Version 2.0 (the "License"); |
-# you may not use this file except in compliance with the License. |
-# You may obtain a copy of the License at |
-# |
-# http://www.apache.org/licenses/LICENSE-2.0 |
-# |
-# Unless required by applicable law or agreed to in writing, software |
-# distributed under the License is distributed on an "AS IS" BASIS, |
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
-# See the License for the specific language governing permissions and |
-# limitations under the License. |
-"""Gsutil API for interacting with cloud storage providers.""" |
- |
-from __future__ import absolute_import |
- |
- |
-class CloudApi(object): |
- """Abstract base class for interacting with cloud storage providers. |
- |
- Implementations of the gsutil Cloud API are not guaranteed to be thread-safe. |
- Behavior when calling a gsutil Cloud API instance simultaneously across |
- threads is undefined and doing so will likely cause errors. Therefore, |
- a separate instance of the gsutil Cloud API should be instantiated per-thread. |
- """ |
- |
- def __init__(self, bucket_storage_uri_class, logger, provider=None, debug=0): |
- """Performs necessary setup for interacting with the cloud storage provider. |
- |
- Args: |
- bucket_storage_uri_class: boto storage_uri class, used by APIs that |
- provide boto translation or mocking. |
- logger: logging.logger for outputting log messages. |
- provider: Default provider prefix describing cloud storage provider to |
- connect to. |
- debug: Debug level for the API implementation (0..3). |
- """ |
- self.bucket_storage_uri_class = bucket_storage_uri_class |
- self.logger = logger |
- self.provider = provider |
- self.debug = debug |
- |
- def GetBucket(self, bucket_name, provider=None, fields=None): |
- """Gets Bucket metadata. |
- |
- Args: |
- bucket_name: Name of the bucket. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Bucket metadata fields, for |
- example, ['logging', 'defaultObjectAcl'] |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Bucket object. |
- """ |
- raise NotImplementedError('GetBucket must be overloaded') |
- |
- def ListBuckets(self, project_id=None, provider=None, fields=None): |
- """Lists bucket metadata for the given project. |
- |
- Args: |
- project_id: Project owning the buckets, default from config if None. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these metadata fields for the listing, |
- for example: |
- ['items/logging', 'items/defaultObjectAcl']. |
- Note that the WildcardIterator class should be used to list |
- buckets instead of calling this function directly. It amends |
- the fields definition from get-like syntax such as |
- ['logging', 'defaultObjectAcl'] so that the caller does not |
- need to prepend 'items/' or specify fields necessary for listing |
- (like nextPageToken). |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Iterator over Bucket objects. |
- """ |
- raise NotImplementedError('ListBuckets must be overloaded') |
- |
- def PatchBucket(self, bucket_name, metadata, canned_acl=None, |
- canned_def_acl=None, preconditions=None, provider=None, |
- fields=None): |
- """Updates bucket metadata for the bucket with patch semantics. |
- |
- Args: |
- bucket_name: Name of bucket to update. |
- metadata: Bucket object defining metadata to be updated. |
- canned_acl: Canned ACL to apply to the bucket. |
- canned_def_acl: Canned default object ACL to apply to the bucket. |
- preconditions: Preconditions for the request. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Bucket metadata fields. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Bucket object describing new bucket metadata. |
- """ |
- raise NotImplementedError('PatchBucket must be overloaded') |
- |
- def CreateBucket(self, bucket_name, project_id=None, metadata=None, |
- provider=None, fields=None): |
- """Creates a new bucket with the specified metadata. |
- |
- Args: |
- bucket_name: Name of the new bucket. |
- project_id: Project owner of the new bucket, default from config if None. |
- metadata: Bucket object defining new bucket metadata. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Bucket metadata fields. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Bucket object describing new bucket metadata. |
- """ |
- raise NotImplementedError('CreateBucket must be overloaded') |
- |
- def DeleteBucket(self, bucket_name, preconditions=None, provider=None): |
- """Deletes a bucket. |
- |
- Args: |
- bucket_name: Name of the bucket to delete. |
- preconditions: Preconditions for the request. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- None. |
- """ |
- raise NotImplementedError('DeleteBucket must be overloaded') |
- |
- class CsObjectOrPrefixType(object): |
- """Enum class for describing CsObjectOrPrefix types.""" |
- OBJECT = 'object' # Cloud object |
- PREFIX = 'prefix' # Cloud bucket subdirectory |
- |
- class CsObjectOrPrefix(object): |
- """Container class for ListObjects results.""" |
- |
- def __init__(self, data, datatype): |
- """Stores a ListObjects result. |
- |
- Args: |
- data: Root object, either an apitools Object or a string Prefix. |
- datatype: CsObjectOrPrefixType of data. |
- """ |
- self.data = data |
- self.datatype = datatype |
- |
- def ListObjects(self, bucket_name, prefix=None, delimiter=None, |
- all_versions=None, provider=None, fields=None): |
- """Lists objects (with metadata) and prefixes in a bucket. |
- |
- Args: |
- bucket_name: Bucket containing the objects. |
- prefix: Prefix for directory-like behavior. |
- delimiter: Delimiter for directory-like behavior. |
- all_versions: If true, list all object versions. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these metadata fields for the listing, |
- for example: |
- ['items/acl', 'items/updated', 'prefixes']. |
- Note that the WildcardIterator class should be used to list |
- objects instead of calling this function directly. It amends |
- the fields definition from get-like syntax such as |
- ['acl', 'updated'] so that the caller does not need to |
- prepend 'items/' or specify any fields necessary for listing |
- (such as prefixes or nextPageToken). |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Iterator over CsObjectOrPrefix wrapper class. |
- """ |
- raise NotImplementedError('ListObjects must be overloaded') |
- |
- def GetObjectMetadata(self, bucket_name, object_name, generation=None, |
- provider=None, fields=None): |
- """Gets object metadata. |
- |
- Args: |
- bucket_name: Bucket containing the object. |
- object_name: Object name. |
- generation: Generation of the object to retrieve. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Object metadata fields, for |
- example, ['acl', 'updated']. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Object object. |
- """ |
- raise NotImplementedError('GetObjectMetadata must be overloaded') |
- |
- def PatchObjectMetadata(self, bucket_name, object_name, metadata, |
- canned_acl=None, generation=None, preconditions=None, |
- provider=None, fields=None): |
- """Updates object metadata with patch semantics. |
- |
- Args: |
- bucket_name: Bucket containing the object. |
- object_name: Object name for object. |
- metadata: Object object defining metadata to be updated. |
- canned_acl: Canned ACL to be set on the object. |
- generation: Generation (or version) of the object to update. |
- preconditions: Preconditions for the request. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Object metadata fields. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Updated object metadata. |
- """ |
- raise NotImplementedError('PatchObjectMetadata must be overloaded') |
- |
- class DownloadStrategy(object): |
- """Enum class for specifying download strategy.""" |
- ONE_SHOT = 'oneshot' |
- RESUMABLE = 'resumable' |
- |
- def GetObjectMedia(self, bucket_name, object_name, download_stream, |
- provider=None, generation=None, object_size=None, |
- download_strategy=DownloadStrategy.ONE_SHOT, start_byte=0, |
- end_byte=None, progress_callback=None, |
- serialization_data=None, digesters=None): |
- """Gets object data. |
- |
- Args: |
- bucket_name: Bucket containing the object. |
- object_name: Object name. |
- download_stream: Stream to send the object data to. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- generation: Generation of the object to retrieve. |
- object_size: Total size of the object being downloaded. |
- download_strategy: Cloud API download strategy to use for download. |
- start_byte: Starting point for download (for resumable downloads and |
- range requests). Can be set to negative to request a range |
- of bytes (python equivalent of [:-3]) |
- end_byte: Ending point for download (for range requests). |
- progress_callback: Optional callback function for progress notifications. |
- Receives calls with arguments |
- (bytes_transferred, total_size). |
- serialization_data: Implementation-specific dict containing serialization |
- information for the download. |
- digesters: Dict of {string : digester}, where string is a name of a hash |
- algorithm, and digester is a validation digester that supports |
- update(bytes) and digest() using that algorithm. |
- Implementation can set the digester value to None to indicate |
- bytes were not successfully digested on-the-fly. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Content-encoding string if it was detected that the server sent an encoded |
- object during transfer, None otherwise. |
- """ |
- raise NotImplementedError('GetObjectMedia must be overloaded') |
- |
- def UploadObject(self, upload_stream, object_metadata, canned_acl=None, |
- size=None, preconditions=None, progress_callback=None, |
- provider=None, fields=None): |
- """Uploads object data and metadata. |
- |
- Args: |
- upload_stream: Seekable stream of object data. |
- object_metadata: Object metadata for new object. Must include bucket |
- and object name. |
- canned_acl: Optional canned ACL to apply to object. Overrides ACL set |
- in object_metadata. |
- size: Optional object size. |
- preconditions: Preconditions for the request. |
- progress_callback: Optional callback function for progress notifications. |
- Receives calls with arguments |
- (bytes_transferred, total_size). |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Object metadata fields. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Object object for newly created destination object. |
- """ |
- raise NotImplementedError('UploadObject must be overloaded') |
- |
- def UploadObjectStreaming(self, upload_stream, object_metadata, |
- canned_acl=None, preconditions=None, |
- progress_callback=None, provider=None, |
- fields=None): |
- """Uploads object data and metadata. |
- |
- Args: |
- upload_stream: Stream of object data. May not be seekable. |
- object_metadata: Object metadata for new object. Must include bucket |
- and object name. |
- canned_acl: Optional canned ACL to apply to object. Overrides ACL set |
- in object_metadata. |
- preconditions: Preconditions for the request. |
- progress_callback: Optional callback function for progress notifications. |
- Receives calls with arguments |
- (bytes_transferred, total_size), but fills in only |
- bytes_transferred. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Object metadata fields. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Object object for newly created destination object. |
- """ |
- raise NotImplementedError('UploadObjectStreaming must be overloaded') |
- |
- def UploadObjectResumable( |
- self, upload_stream, object_metadata, canned_acl=None, |
- size=None, preconditions=None, serialization_data=None, |
- tracker_callback=None, progress_callback=None, provider=None, |
- fields=None): |
- """Uploads object data and metadata using a resumable upload strategy. |
- |
- Args: |
- upload_stream: Seekable stream of object data. |
- object_metadata: Object metadata for new object. Must include bucket |
- and object name. |
- canned_acl: Optional canned ACL to apply to object. Overrides ACL set |
- in object_metadata. |
- size: Total size of the object. |
- preconditions: Preconditions for the request. |
- serialization_data: Dict of {'url' : UploadURL} allowing for uploads to |
- be resumed. |
- tracker_callback: Callback function taking a upload URL string. |
- Guaranteed to be called when the implementation gets an |
- upload URL, allowing the caller to resume the upload |
- across process breaks by saving the upload URL in |
- a tracker file. |
- progress_callback: Optional callback function for progress notifications. |
- Receives calls with arguments |
- (bytes_transferred, total_size). |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Object metadata fields when the |
- upload is complete. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Object object for newly created destination object. |
- """ |
- raise NotImplementedError('UploadObjectResumable must be overloaded') |
- |
- def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, |
- canned_acl=None, preconditions=None, progress_callback=None, |
- max_bytes_per_call=None, provider=None, fields=None): |
- """Copies an object in the cloud. |
- |
- Args: |
- src_obj_metadata: Object metadata for source object. Must include |
- bucket name, object name, and etag. |
- dst_obj_metadata: Object metadata for new object. Must include bucket |
- and object name. |
- src_generation: Generation of the source object to copy. |
- canned_acl: Optional canned ACL to apply to destination object. Overrides |
- ACL set in dst_obj_metadata. |
- preconditions: Destination object preconditions for the request. |
- progress_callback: Optional callback function for progress notifications. |
- Receives calls with arguments |
- (bytes_transferred, total_size). |
- max_bytes_per_call: Integer describing maximum number of bytes |
- to rewrite per service call. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Object metadata fields. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Object object for newly created destination object. |
- """ |
- raise NotImplementedError('CopyObject must be overloaded') |
- |
- def ComposeObject(self, src_objs_metadata, dst_obj_metadata, |
- preconditions=None, provider=None, fields=None): |
- """Composes an object in the cloud. |
- |
- Args: |
- src_objs_metadata: List of ComposeRequest.SourceObjectsValueListEntries |
- specifying the objects to compose. |
- dst_obj_metadata: Metadata for the destination object including bucket |
- and object name. |
- preconditions: Destination object preconditions for the request. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Object metadata fields. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Composed object metadata. |
- """ |
- raise NotImplementedError('ComposeObject must be overloaded') |
- |
- def DeleteObject(self, bucket_name, object_name, preconditions=None, |
- generation=None, provider=None): |
- """Deletes an object. |
- |
- Args: |
- bucket_name: Name of the containing bucket. |
- object_name: Name of the object to delete. |
- preconditions: Preconditions for the request. |
- generation: Generation (or version) of the object to delete; if None, |
- deletes the live object. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- None. |
- """ |
- raise NotImplementedError('DeleteObject must be overloaded') |
- |
- def WatchBucket(self, bucket_name, address, channel_id, token=None, |
- provider=None, fields=None): |
- """Creates a notification subscription for changes to objects in a bucket. |
- |
- Args: |
- bucket_name: Bucket containing the objects. |
- address: Address to which to send notifications. |
- channel_id: Unique ID string for the channel. |
- token: If present, token string is delivered with each notification. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- fields: If present, return only these Channel metadata fields. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- Channel object describing the notification subscription. |
- """ |
- raise NotImplementedError('WatchBucket must be overloaded') |
- |
- def StopChannel(self, channel_id, resource_id, provider=None): |
- """Stops a notification channel. |
- |
- Args: |
- channel_id: Unique ID string for the channel. |
- resource_id: Version-agnostic ID string for the channel. |
- provider: Cloud storage provider to connect to. If not present, |
- class-wide default is used. |
- |
- Raises: |
- ArgumentException for errors during input validation. |
- ServiceException for errors interacting with cloud storage providers. |
- |
- Returns: |
- None. |
- """ |
- raise NotImplementedError('StopChannel must be overloaded') |
- |
- |
-class Preconditions(object): |
- """Preconditions class for specifying preconditions to cloud API requests.""" |
- |
- def __init__(self, gen_match=None, meta_gen_match=None): |
- """Instantiates a Preconditions object. |
- |
- Args: |
- gen_match: Perform request only if generation of target object |
- matches the given integer. Ignored for bucket requests. |
- meta_gen_match: Perform request only if metageneration of target |
- object/bucket matches the given integer. |
- """ |
- self.gen_match = gen_match |
- self.meta_gen_match = meta_gen_match |
- |
- |
-class ArgumentException(Exception): |
- """Exception raised when arguments to a Cloud API method are invalid. |
- |
- This exception is never raised as a result of a failed call to a cloud |
- storage provider. |
- """ |
- |
- def __init__(self, reason): |
- Exception.__init__(self) |
- self.reason = reason |
- |
- def __repr__(self): |
- return str(self) |
- |
- def __str__(self): |
- return '%s: %s' % (self.__class__.__name__, self.reason) |
- |
- |
-class ProjectIdException(ArgumentException): |
- """Exception raised when a Project ID argument is required but not present.""" |
- |
- |
-class ServiceException(Exception): |
- """Exception raised when a cloud storage provider request fails. |
- |
- This exception is raised only as a result of a failed remote call. |
- """ |
- |
- def __init__(self, reason, status=None, body=None): |
- Exception.__init__(self) |
- self.reason = reason |
- self.status = status |
- self.body = body |
- |
- def __repr__(self): |
- return str(self) |
- |
- def __str__(self): |
- message = '%s:' % self.__class__.__name__ |
- if self.status: |
- message += ' %s' % self.status |
- message += ' %s' % self.reason |
- if self.body: |
- message += '\n%s' % self.body |
- return message |
- |
- |
-class RetryableServiceException(ServiceException): |
- """Exception class for retryable exceptions.""" |
- |
- |
-class ResumableDownloadException(RetryableServiceException): |
- """Exception raised for res. downloads that can be retried later.""" |
- |
- |
-class ResumableUploadException(RetryableServiceException): |
- """Exception raised for res. uploads that can be retried w/ same upload ID.""" |
- |
- |
-class ResumableUploadStartOverException(RetryableServiceException): |
- """Exception raised for res. uploads that can be retried w/ new upload ID.""" |
- |
- |
-class ResumableUploadAbortException(ServiceException): |
- """Exception raised for resumable uploads that cannot be retried later.""" |
- |
- |
-class AuthenticationException(ServiceException): |
- """Exception raised for errors during the authentication process.""" |
- |
- |
-class PreconditionException(ServiceException): |
- """Exception raised for precondition failures.""" |
- |
- |
-class NotFoundException(ServiceException): |
- """Exception raised when a resource is not found (404).""" |
- |
- |
-class NotEmptyException(ServiceException): |
- """Exception raised when trying to delete a bucket is not empty.""" |
- |
- |
-class BadRequestException(ServiceException): |
- """Exception raised for malformed requests. |
- |
- Where it is possible to detect invalid arguments prior to sending them |
- to the server, an ArgumentException should be raised instead. |
- """ |
- |
- |
-class AccessDeniedException(ServiceException): |
- """Exception raised when authenticated user has insufficient access rights. |
- |
- This is raised when the authentication process succeeded but the |
- authenticated user does not have access rights to the requested resource. |
- """ |
- |
- |