Revert "Add gsutil 4.13 to telemetry/third_party"

tools/telemetry/third_party/gsutil/gslib/commands/setmeta.py

-# -*- coding: utf-8 -*-
-# Copyright 2012 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Implementation of setmeta command for setting cloud object metadata."""
-
-from __future__ import absolute_import
-
-from gslib.cloud_api import AccessDeniedException
-from gslib.cloud_api import PreconditionException
-from gslib.cloud_api import Preconditions
-from gslib.command import Command
-from gslib.command_argument import CommandArgument
-from gslib.cs_api_map import ApiSelector
-from gslib.exception import CommandException
-from gslib.name_expansion import NameExpansionIterator
-from gslib.storage_url import StorageUrlFromString
-from gslib.translation_helper import CopyObjectMetadata
-from gslib.translation_helper import ObjectMetadataFromHeaders
-from gslib.translation_helper import PreconditionsFromHeaders
-from gslib.util import GetCloudApiInstance
-from gslib.util import NO_MAX
-from gslib.util import Retry
-
-
-_SYNOPSIS = """
-  gsutil setmeta -h [header:value|header] ... url...
-"""
-
-_DETAILED_HELP_TEXT = ("""
-<B>SYNOPSIS</B>
-""" + _SYNOPSIS + """
-
-
-<B>DESCRIPTION</B>
-  The gsutil setmeta command allows you to set or remove the metadata on one
-  or more objects. It takes one or more header arguments followed by one or
-  more URLs, where each header argument is in one of two forms:
-
-  - if you specify header:value, it will set the given header on all
-    named objects.
-
-  - if you specify header (with no value), it will remove the given header
-    from all named objects.
-
-  For example, the following command would set the Content-Type and
-  Cache-Control and remove the Content-Disposition on the specified objects:
-
-    gsutil setmeta -h "Content-Type:text/html" \\
-      -h "Cache-Control:public, max-age=3600" \\
-      -h "Content-Disposition" gs://bucket/*.html
-
-  If you have a large number of objects to update you might want to use the
-  gsutil -m option, to perform a parallel (multi-threaded/multi-processing)
-  update:
-
-    gsutil -m setmeta -h "Content-Type:text/html" \\
-      -h "Cache-Control:public, max-age=3600" \\
-      -h "Content-Disposition" gs://bucket/*.html
-
-  You can also use the setmeta command to set custom metadata on an object:
-
-    gsutil setmeta -h "x-goog-meta-icecreamflavor:vanilla" gs://bucket/object
-
-  See "gsutil help metadata" for details about how you can set metadata
-  while uploading objects, what metadata fields can be set and the meaning of
-  these fields, use of custom metadata, and how to view currently set metadata.
-
-  NOTE: By default, publicly readable objects are served with a Cache-Control
-  header allowing such objects to be cached for 3600 seconds. For more details
-  about this default behavior see the CACHE-CONTROL section of
-  "gsutil help metadata". If you need to ensure that updates become visible
-  immediately, you should set a Cache-Control header of "Cache-Control:private,
-  max-age=0, no-transform" on such objects.  You can do this with the command:
-
-    gsutil setmeta -h "Content-Type:text/html" \\
-      -h "Cache-Control:private, max-age=0, no-transform" gs://bucket/*.html
-
-  The setmeta command reads each object's current generation and metageneration
-  and uses those as preconditions unless they are otherwise specified by
-  top-level arguments. For example:
-
-    gsutil -h "x-goog-if-metageneration-match:2" setmeta
-      -h "x-goog-meta-icecreamflavor:vanilla"
-
-  will set the icecreamflavor:vanilla metadata if the current live object has a
-  metageneration of 2.
-
-<B>OPTIONS</B>
-  -h          Specifies a header:value to be added, or header to be removed,
-              from each named object.
-""")
-
-# Setmeta assumes a header-like model which doesn't line up with the JSON way
-# of doing things. This list comes from functionality that was supported by
-# gsutil3 at the time gsutil4 was released.
-SETTABLE_FIELDS = ['cache-control', 'content-disposition',
-                   'content-encoding', 'content-language',
-                   'content-md5', 'content-type']
-
-
-def _SetMetadataExceptionHandler(cls, e):
-  """Exception handler that maintains state about post-completion status."""
-  cls.logger.error(e)
-  cls.everything_set_okay = False
-
-
-def _SetMetadataFuncWrapper(cls, name_expansion_result, thread_state=None):
-  cls.SetMetadataFunc(name_expansion_result, thread_state=thread_state)
-
-
-class SetMetaCommand(Command):
-  """Implementation of gsutil setmeta command."""
-
-  # Command specification. See base class for documentation.
-  command_spec = Command.CreateCommandSpec(
-      'setmeta',
-      command_name_aliases=['setheader'],
-      usage_synopsis=_SYNOPSIS,
-      min_args=1,
-      max_args=NO_MAX,
-      supported_sub_args='h:rR',
-      file_url_ok=False,
-      provider_url_ok=False,
-      urls_start_arg=1,
-      gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
-      gs_default_api=ApiSelector.JSON,
-      argparse_arguments=[
-          CommandArgument.MakeZeroOrMoreCloudURLsArgument()
-      ]
-  )
-  # Help specification. See help_provider.py for documentation.
-  help_spec = Command.HelpSpec(
-      help_name='setmeta',
-      help_name_aliases=['setheader'],
-      help_type='command_help',
-      help_one_line_summary='Set metadata on already uploaded objects',
-      help_text=_DETAILED_HELP_TEXT,
-      subcommand_help_text={},
-  )
-
-  def RunCommand(self):
-    """Command entry point for the setmeta command."""
-    headers = []
-    if self.sub_opts:
-      for o, a in self.sub_opts:
-        if o == '-h':
-          if 'x-goog-acl' in a or 'x-amz-acl' in a:
-            raise CommandException(
-                'gsutil setmeta no longer allows canned ACLs. Use gsutil acl '
-                'set ... to set canned ACLs.')
-          headers.append(a)
-
-    (metadata_minus, metadata_plus) = self._ParseMetadataHeaders(headers)
-
-    self.metadata_change = metadata_plus
-    for header in metadata_minus:
-      self.metadata_change[header] = ''
-
-    if len(self.args) == 1 and not self.recursion_requested:
-      url = StorageUrlFromString(self.args[0])
-      if not (url.IsCloudUrl() and url.IsObject()):
-        raise CommandException('URL (%s) must name an object' % self.args[0])
-
-    # Used to track if any objects' metadata failed to be set.
-    self.everything_set_okay = True
-
-    self.preconditions = PreconditionsFromHeaders(self.headers)
-
-    name_expansion_iterator = NameExpansionIterator(
-        self.command_name, self.debug, self.logger, self.gsutil_api,
-        self.args, self.recursion_requested, all_versions=self.all_versions,
-        continue_on_error=self.parallel_operations)
-
-    try:
-      # Perform requests in parallel (-m) mode, if requested, using
-      # configured number of parallel processes and threads. Otherwise,
-      # perform requests with sequential function calls in current process.
-      self.Apply(_SetMetadataFuncWrapper, name_expansion_iterator,
-                 _SetMetadataExceptionHandler, fail_on_error=True)
-    except AccessDeniedException as e:
-      if e.status == 403:
-        self._WarnServiceAccounts()
-      raise
-
-    if not self.everything_set_okay:
-      raise CommandException('Metadata for some objects could not be set.')
-
-    return 0
-
-  @Retry(PreconditionException, tries=3, timeout_secs=1)
-  def SetMetadataFunc(self, name_expansion_result, thread_state=None):
-    """Sets metadata on an object.
-
-    Args:
-      name_expansion_result: NameExpansionResult describing target object.
-      thread_state: gsutil Cloud API instance to use for the operation.
-    """
-    gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)
-
-    exp_src_url = name_expansion_result.expanded_storage_url
-    self.logger.info('Setting metadata on %s...', exp_src_url)
-
-    fields = ['generation', 'metadata', 'metageneration']
-    cloud_obj_metadata = gsutil_api.GetObjectMetadata(
-        exp_src_url.bucket_name, exp_src_url.object_name,
-        generation=exp_src_url.generation, provider=exp_src_url.scheme,
-        fields=fields)
-
-    preconditions = Preconditions(
-        gen_match=self.preconditions.gen_match,
-        meta_gen_match=self.preconditions.meta_gen_match)
-    if preconditions.gen_match is None:
-      preconditions.gen_match = cloud_obj_metadata.generation
-    if preconditions.meta_gen_match is None:
-      preconditions.meta_gen_match = cloud_obj_metadata.metageneration
-
-    # Patch handles the patch semantics for most metadata, but we need to
-    # merge the custom metadata field manually.
-    patch_obj_metadata = ObjectMetadataFromHeaders(self.metadata_change)
-
-    api = gsutil_api.GetApiSelector(provider=exp_src_url.scheme)
-    # For XML we only want to patch through custom metadata that has
-    # changed.  For JSON we need to build the complete set.
-    if api == ApiSelector.XML:
-      pass
-    elif api == ApiSelector.JSON:
-      CopyObjectMetadata(patch_obj_metadata, cloud_obj_metadata,
-                         override=True)
-      patch_obj_metadata = cloud_obj_metadata
-      # Patch body does not need the object generation and metageneration.
-      patch_obj_metadata.generation = None
-      patch_obj_metadata.metageneration = None
-
-    gsutil_api.PatchObjectMetadata(
-        exp_src_url.bucket_name, exp_src_url.object_name, patch_obj_metadata,
-        generation=exp_src_url.generation, preconditions=preconditions,
-        provider=exp_src_url.scheme)
-
-  def _ParseMetadataHeaders(self, headers):
-    """Validates and parses metadata changes from the headers argument.
-
-    Args:
-      headers: Header dict to validate and parse.
-
-    Returns:
-      (metadata_plus, metadata_minus): Tuple of header sets to add and remove.
-    """
-    metadata_minus = set()
-    cust_metadata_minus = set()
-    metadata_plus = {}
-    cust_metadata_plus = {}
-    # Build a count of the keys encountered from each plus and minus arg so we
-    # can check for dupe field specs.
-    num_metadata_plus_elems = 0
-    num_cust_metadata_plus_elems = 0
-    num_metadata_minus_elems = 0
-    num_cust_metadata_minus_elems = 0
-
-    for md_arg in headers:
-      parts = md_arg.split(':')
-      if len(parts) not in (1, 2):
-        raise CommandException(
-            'Invalid argument: must be either header or header:value (%s)' %
-            md_arg)
-      if len(parts) == 2:
-        (header, value) = parts
-      else:
-        (header, value) = (parts[0], None)
-      _InsistAsciiHeader(header)
-      # Translate headers to lowercase to match the casing assumed by our
-      # sanity-checking operations.
-      header = header.lower()
-      if value:
-        if _IsCustomMeta(header):
-          # Allow non-ASCII data for custom metadata fields.
-          cust_metadata_plus[header] = value
-          num_cust_metadata_plus_elems += 1
-        else:
-          # Don't unicode encode other fields because that would perturb their
-          # content (e.g., adding %2F's into the middle of a Cache-Control
-          # value).
-          _InsistAsciiHeaderValue(header, value)
-          value = str(value)
-          metadata_plus[header] = value
-          num_metadata_plus_elems += 1
-      else:
-        if _IsCustomMeta(header):
-          cust_metadata_minus.add(header)
-          num_cust_metadata_minus_elems += 1
-        else:
-          metadata_minus.add(header)
-          num_metadata_minus_elems += 1
-
-    if (num_metadata_plus_elems != len(metadata_plus)
-        or num_cust_metadata_plus_elems != len(cust_metadata_plus)
-        or num_metadata_minus_elems != len(metadata_minus)
-        or num_cust_metadata_minus_elems != len(cust_metadata_minus)
-        or metadata_minus.intersection(set(metadata_plus.keys()))):
-      raise CommandException('Each header must appear at most once.')
-    other_than_base_fields = (set(metadata_plus.keys())
-                              .difference(SETTABLE_FIELDS))
-    other_than_base_fields.update(
-        metadata_minus.difference(SETTABLE_FIELDS))
-    for f in other_than_base_fields:
-      # This check is overly simple; it would be stronger to check, for each
-      # URL argument, whether f.startswith the
-      # provider metadata_prefix, but here we just parse the spec
-      # once, before processing any of the URLs. This means we will not
-      # detect if the user tries to set an x-goog-meta- field on an another
-      # provider's object, for example.
-      if not _IsCustomMeta(f):
-        raise CommandException(
-            'Invalid or disallowed header (%s).\nOnly these fields (plus '
-            'x-goog-meta-* fields) can be set or unset:\n%s' % (
-                f, sorted(list(SETTABLE_FIELDS))))
-    metadata_plus.update(cust_metadata_plus)
-    metadata_minus.update(cust_metadata_minus)
-    return (metadata_minus, metadata_plus)
-
-
-def _InsistAscii(string, message):
-  if not all(ord(c) < 128 for c in string):
-    raise CommandException(message)
-
-
-def _InsistAsciiHeader(header):
-  _InsistAscii(header, 'Invalid non-ASCII header (%s).' % header)
-
-
-def _InsistAsciiHeaderValue(header, value):
-  _InsistAscii(
-      value, ('Invalid non-ASCII value (%s) was provided for header %s.'
-              % (value, header)))
-
-
-def _IsCustomMeta(header):
-  return header.startswith('x-goog-meta-') or header.startswith('x-amz-meta-')