[catapult] - Copy Telemetry's gsutilz over to third_party.

third_party/gsutil/gslib/commands/compose.py

+# -*- coding: utf-8 -*-
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of compose command for Google Cloud Storage."""
+
+from __future__ import absolute_import
+
+from gslib.bucket_listing_ref import BucketListingObject
+from gslib.command import Command
+from gslib.command_argument import CommandArgument
+from gslib.cs_api_map import ApiSelector
+from gslib.exception import CommandException
+from gslib.storage_url import ContainsWildcard
+from gslib.storage_url import StorageUrlFromString
+from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
+from gslib.translation_helper import PreconditionsFromHeaders
+
+MAX_COMPONENT_COUNT = 1024
+MAX_COMPOSE_ARITY = 32
+
+_SYNOPSIS = """
+  gsutil compose gs://bucket/obj1 gs://bucket/obj2 ... gs://bucket/composite
+"""
+
+_DETAILED_HELP_TEXT = ("""
+<B>SYNOPSIS</B>
+""" + _SYNOPSIS + """
+
+
+<B>DESCRIPTION</B>
+  The compose command creates a new object whose content is the concatenation
+  of a given sequence of component objects under the same bucket. gsutil uses
+  the content type of the first source object to determine the destination
+  object's content type. For more information, please see:
+  https://developers.google.com/storage/docs/composite-objects
+
+  Note also that the gsutil cp command will automatically split uploads for
+  large files into multiple component objects, upload them in parallel, and
+  compose them into a final object (which will be subject to the component
+  count limit). This will still perform all uploads from a single machine. For
+  extremely large files and/or very low per-machine bandwidth, you may want to
+  split the file and upload it from multiple machines, and later compose these
+  parts of the file manually. See the 'PARALLEL COMPOSITE UPLOADS' section under
+  'gsutil help cp' for details.
+
+  Appending simply entails uploading your new data to a temporary object,
+  composing it with the growing append-target, and deleting the temporary
+  object:
+
+    $ echo 'new data' | gsutil cp - gs://bucket/data-to-append
+    $ gsutil compose gs://bucket/append-target gs://bucket/data-to-append \\
+        gs://bucket/append-target
+    $ gsutil rm gs://bucket/data-to-append
+
+  Note that there is a limit (currently %d) to the number of components for a
+  given composite object. This means you can append to each object at most %d
+  times.
+""" % (MAX_COMPONENT_COUNT, MAX_COMPONENT_COUNT - 1))
+
+
+class ComposeCommand(Command):
+  """Implementation of gsutil compose command."""
+
+  # Command specification. See base class for documentation.
+  command_spec = Command.CreateCommandSpec(
+      'compose',
+      command_name_aliases=['concat'],
+      usage_synopsis=_SYNOPSIS,
+      min_args=2,
+      max_args=MAX_COMPOSE_ARITY + 1,
+      supported_sub_args='',
+      # Not files, just object names without gs:// prefix.
+      file_url_ok=False,
+      provider_url_ok=False,
+      urls_start_arg=1,
+      gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
+      gs_default_api=ApiSelector.JSON,
+      argparse_arguments=[
+          CommandArgument.MakeZeroOrMoreCloudURLsArgument()
+      ]
+  )
+  # Help specification. See help_provider.py for documentation.
+  help_spec = Command.HelpSpec(
+      help_name='compose',
+      help_name_aliases=['concat'],
+      help_type='command_help',
+      help_one_line_summary=(
+          'Concatenate a sequence of objects into a new composite object.'),
+      help_text=_DETAILED_HELP_TEXT,
+      subcommand_help_text={},
+  )
+
+  def CheckProvider(self, url):
+    if url.scheme != 'gs':
+      raise CommandException(
+          '"compose" called on URL with unsupported provider (%s).' % str(url))
+
+  # Command entry point.
+  def RunCommand(self):
+    """Command entry point for the compose command."""
+    target_url_str = self.args[-1]
+    self.args = self.args[:-1]
+    target_url = StorageUrlFromString(target_url_str)
+    self.CheckProvider(target_url)
+    if target_url.HasGeneration():
+      raise CommandException('A version-specific URL (%s) cannot be '
+                             'the destination for gsutil compose - abort.'
+                             % target_url)
+
+    dst_obj_metadata = apitools_messages.Object(name=target_url.object_name,
+                                                bucket=target_url.bucket_name)
+
+    components = []
+    # Remember the first source object so we can get its content type.
+    first_src_url = None
+    for src_url_str in self.args:
+      if ContainsWildcard(src_url_str):
+        src_url_iter = self.WildcardIterator(src_url_str).IterObjects()
+      else:
+        src_url_iter = [BucketListingObject(StorageUrlFromString(src_url_str))]
+      for blr in src_url_iter:
+        src_url = blr.storage_url
+        self.CheckProvider(src_url)
+
+        if src_url.bucket_name != target_url.bucket_name:
+          raise CommandException(
+              'GCS does not support inter-bucket composing.')
+
+        if not first_src_url:
+          first_src_url = src_url
+        src_obj_metadata = (
+            apitools_messages.ComposeRequest.SourceObjectsValueListEntry(
+                name=src_url.object_name))
+        if src_url.HasGeneration():
+          src_obj_metadata.generation = src_url.generation
+        components.append(src_obj_metadata)
+        # Avoid expanding too many components, and sanity check each name
+        # expansion result.
+        if len(components) > MAX_COMPOSE_ARITY:
+          raise CommandException('"compose" called with too many component '
+                                 'objects. Limit is %d.' % MAX_COMPOSE_ARITY)
+
+    if len(components) < 2:
+      raise CommandException('"compose" requires at least 2 component objects.')
+
+    dst_obj_metadata.contentType = self.gsutil_api.GetObjectMetadata(
+        first_src_url.bucket_name, first_src_url.object_name,
+        provider=first_src_url.scheme, fields=['contentType']).contentType
+
+    preconditions = PreconditionsFromHeaders(self.headers or {})
+
+    self.logger.info(
+        'Composing %s from %d component objects.', target_url, len(components))
+    self.gsutil_api.ComposeObject(components, dst_obj_metadata,
+                                  preconditions=preconditions,
+                                  provider=target_url.scheme)