| Index: tools/telemetry/third_party/gsutilz/gslib/commands/compose.py
|
| diff --git a/tools/telemetry/third_party/gsutilz/gslib/commands/compose.py b/tools/telemetry/third_party/gsutilz/gslib/commands/compose.py
|
| deleted file mode 100644
|
| index e33261fe8ce168118ab8fdc50e070a3cc306c023..0000000000000000000000000000000000000000
|
| --- a/tools/telemetry/third_party/gsutilz/gslib/commands/compose.py
|
| +++ /dev/null
|
| @@ -1,167 +0,0 @@
|
| -# -*- coding: utf-8 -*-
|
| -# Copyright 2013 Google Inc. All Rights Reserved.
|
| -#
|
| -# Licensed under the Apache License, Version 2.0 (the "License");
|
| -# you may not use this file except in compliance with the License.
|
| -# You may obtain a copy of the License at
|
| -#
|
| -# http://www.apache.org/licenses/LICENSE-2.0
|
| -#
|
| -# Unless required by applicable law or agreed to in writing, software
|
| -# distributed under the License is distributed on an "AS IS" BASIS,
|
| -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| -# See the License for the specific language governing permissions and
|
| -# limitations under the License.
|
| -"""Implementation of compose command for Google Cloud Storage."""
|
| -
|
| -from __future__ import absolute_import
|
| -
|
| -from gslib.bucket_listing_ref import BucketListingObject
|
| -from gslib.command import Command
|
| -from gslib.command_argument import CommandArgument
|
| -from gslib.cs_api_map import ApiSelector
|
| -from gslib.exception import CommandException
|
| -from gslib.storage_url import ContainsWildcard
|
| -from gslib.storage_url import StorageUrlFromString
|
| -from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
|
| -from gslib.translation_helper import PreconditionsFromHeaders
|
| -
|
| -MAX_COMPONENT_COUNT = 1024
|
| -MAX_COMPOSE_ARITY = 32
|
| -
|
| -_SYNOPSIS = """
|
| - gsutil compose gs://bucket/obj1 gs://bucket/obj2 ... gs://bucket/composite
|
| -"""
|
| -
|
| -_DETAILED_HELP_TEXT = ("""
|
| -<B>SYNOPSIS</B>
|
| -""" + _SYNOPSIS + """
|
| -
|
| -
|
| -<B>DESCRIPTION</B>
|
| - The compose command creates a new object whose content is the concatenation
|
| - of a given sequence of component objects under the same bucket. gsutil uses
|
| - the content type of the first source object to determine the destination
|
| - object's content type. For more information, please see:
|
| - https://developers.google.com/storage/docs/composite-objects
|
| -
|
| - Note also that the gsutil cp command will automatically split uploads for
|
| - large files into multiple component objects, upload them in parallel, and
|
| - compose them into a final object (which will be subject to the component
|
| - count limit). This will still perform all uploads from a single machine. For
|
| - extremely large files and/or very low per-machine bandwidth, you may want to
|
| - split the file and upload it from multiple machines, and later compose these
|
| - parts of the file manually. See the 'PARALLEL COMPOSITE UPLOADS' section under
|
| - 'gsutil help cp' for details.
|
| -
|
| - Appending simply entails uploading your new data to a temporary object,
|
| - composing it with the growing append-target, and deleting the temporary
|
| - object:
|
| -
|
| - $ echo 'new data' | gsutil cp - gs://bucket/data-to-append
|
| - $ gsutil compose gs://bucket/append-target gs://bucket/data-to-append \\
|
| - gs://bucket/append-target
|
| - $ gsutil rm gs://bucket/data-to-append
|
| -
|
| - Note that there is a limit (currently %d) to the number of components for a
|
| - given composite object. This means you can append to each object at most %d
|
| - times.
|
| -""" % (MAX_COMPONENT_COUNT, MAX_COMPONENT_COUNT - 1))
|
| -
|
| -
|
| -class ComposeCommand(Command):
|
| - """Implementation of gsutil compose command."""
|
| -
|
| - # Command specification. See base class for documentation.
|
| - command_spec = Command.CreateCommandSpec(
|
| - 'compose',
|
| - command_name_aliases=['concat'],
|
| - usage_synopsis=_SYNOPSIS,
|
| - min_args=2,
|
| - max_args=MAX_COMPOSE_ARITY + 1,
|
| - supported_sub_args='',
|
| - # Not files, just object names without gs:// prefix.
|
| - file_url_ok=False,
|
| - provider_url_ok=False,
|
| - urls_start_arg=1,
|
| - gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
|
| - gs_default_api=ApiSelector.JSON,
|
| - argparse_arguments=[
|
| - CommandArgument.MakeZeroOrMoreCloudURLsArgument()
|
| - ]
|
| - )
|
| - # Help specification. See help_provider.py for documentation.
|
| - help_spec = Command.HelpSpec(
|
| - help_name='compose',
|
| - help_name_aliases=['concat'],
|
| - help_type='command_help',
|
| - help_one_line_summary=(
|
| - 'Concatenate a sequence of objects into a new composite object.'),
|
| - help_text=_DETAILED_HELP_TEXT,
|
| - subcommand_help_text={},
|
| - )
|
| -
|
| - def CheckProvider(self, url):
|
| - if url.scheme != 'gs':
|
| - raise CommandException(
|
| - '"compose" called on URL with unsupported provider (%s).' % str(url))
|
| -
|
| - # Command entry point.
|
| - def RunCommand(self):
|
| - """Command entry point for the compose command."""
|
| - target_url_str = self.args[-1]
|
| - self.args = self.args[:-1]
|
| - target_url = StorageUrlFromString(target_url_str)
|
| - self.CheckProvider(target_url)
|
| - if target_url.HasGeneration():
|
| - raise CommandException('A version-specific URL (%s) cannot be '
|
| - 'the destination for gsutil compose - abort.'
|
| - % target_url)
|
| -
|
| - dst_obj_metadata = apitools_messages.Object(name=target_url.object_name,
|
| - bucket=target_url.bucket_name)
|
| -
|
| - components = []
|
| - # Remember the first source object so we can get its content type.
|
| - first_src_url = None
|
| - for src_url_str in self.args:
|
| - if ContainsWildcard(src_url_str):
|
| - src_url_iter = self.WildcardIterator(src_url_str).IterObjects()
|
| - else:
|
| - src_url_iter = [BucketListingObject(StorageUrlFromString(src_url_str))]
|
| - for blr in src_url_iter:
|
| - src_url = blr.storage_url
|
| - self.CheckProvider(src_url)
|
| -
|
| - if src_url.bucket_name != target_url.bucket_name:
|
| - raise CommandException(
|
| - 'GCS does not support inter-bucket composing.')
|
| -
|
| - if not first_src_url:
|
| - first_src_url = src_url
|
| - src_obj_metadata = (
|
| - apitools_messages.ComposeRequest.SourceObjectsValueListEntry(
|
| - name=src_url.object_name))
|
| - if src_url.HasGeneration():
|
| - src_obj_metadata.generation = src_url.generation
|
| - components.append(src_obj_metadata)
|
| - # Avoid expanding too many components, and sanity check each name
|
| - # expansion result.
|
| - if len(components) > MAX_COMPOSE_ARITY:
|
| - raise CommandException('"compose" called with too many component '
|
| - 'objects. Limit is %d.' % MAX_COMPOSE_ARITY)
|
| -
|
| - if len(components) < 2:
|
| - raise CommandException('"compose" requires at least 2 component objects.')
|
| -
|
| - dst_obj_metadata.contentType = self.gsutil_api.GetObjectMetadata(
|
| - first_src_url.bucket_name, first_src_url.object_name,
|
| - provider=first_src_url.scheme, fields=['contentType']).contentType
|
| -
|
| - preconditions = PreconditionsFromHeaders(self.headers or {})
|
| -
|
| - self.logger.info(
|
| - 'Composing %s from %d component objects.', target_url, len(components))
|
| - self.gsutil_api.ComposeObject(components, dst_obj_metadata,
|
| - preconditions=preconditions,
|
| - provider=target_url.scheme)
|
|
|