Update checked in version of gsutil to version 4.6

gslib/commands/acl.py

Index: gslib/commands/acl.py
===================================================================
--- gslib/commands/acl.py	(revision 33376)
+++ gslib/commands/acl.py	(working copy)
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2011 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -11,43 +12,39 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""Implementation of acl command for cloud storage providers."""
 
+from __future__ import absolute_import
+
 import getopt
 
-from boto.exception import GSResponseError
 from gslib import aclhelpers
-from gslib import name_expansion
+from gslib.cloud_api import AccessDeniedException
+from gslib.cloud_api import BadRequestException
+from gslib.cloud_api import Preconditions
+from gslib.cloud_api import ServiceException
 from gslib.command import Command
-from gslib.command import COMMAND_NAME
-from gslib.command import COMMAND_NAME_ALIASES
-from gslib.command import FILE_URIS_OK
-from gslib.command import MAX_ARGS
-from gslib.command import MIN_ARGS
-from gslib.command import PROVIDER_URIS_OK
-from gslib.command import SUPPORTED_SUB_ARGS
-from gslib.command import URIS_START_ARG
+from gslib.command import SetAclExceptionHandler
+from gslib.command import SetAclFuncWrapper
+from gslib.cs_api_map import ApiSelector
 from gslib.exception import CommandException
 from gslib.help_provider import CreateHelpText
-from gslib.help_provider import HELP_NAME
-from gslib.help_provider import HELP_NAME_ALIASES
-from gslib.help_provider import HELP_ONE_LINE_SUMMARY
-from gslib.help_provider import HELP_TEXT
-from gslib.help_provider import HELP_TYPE
-from gslib.help_provider import HelpType
-from gslib.help_provider import SUBCOMMAND_HELP_TEXT
+from gslib.storage_url import StorageUrlFromString
+from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
 from gslib.util import NO_MAX
 from gslib.util import Retry
+from gslib.util import UrlsAreForSingleProvider
 
 _SET_SYNOPSIS = """
-  gsutil acl set [-f] [-R] [-a] file-or-canned_acl_name uri...
+  gsutil acl set [-f] [-R] [-a] file-or-canned_acl_name url...
 """
 
 _GET_SYNOPSIS = """
-  gsutil acl get uri
+  gsutil acl get url
 """
 
 _CH_SYNOPSIS = """
-  gsutil acl ch [-R] -u|-g|-d <grant>... uri...
+  gsutil acl ch [-R] -u|-g|-d <grant>... url...
 
   where each <grant> is one of the following forms:
 
@@ -58,7 +55,7 @@
 
 _GET_DESCRIPTION = """
 <B>GET</B>
-  The "acl get" command gets the ACL XML for a bucket or object, which you can
+  The "acl get" command gets the ACL text for a bucket or object, which you can
   save and edit for the acl set command.
 """
 
@@ -104,7 +101,7 @@
 
     gsutil -m acl set acl.txt gs://bucket/*.jpg
 
-  Note that multi-threading/multi-processing is only done when the named URIs
+  Note that multi-threading/multi-processing is only done when the named URLs
   refer to objects. gsutil -m acl set gs://bucket1 gs://bucket2 will run the
   acl set operations sequentially.
 
@@ -113,14 +110,15 @@
   The "set" sub-command has the following options
 
     -R, -r      Performs "acl set" request recursively, to all objects under
-                the specified URI.
+                the specified URL.
 
     -a          Performs "acl set" request on all object versions.
 
     -f          Normally gsutil stops at the first error. The -f option causes
-                it to continue when it encounters errors. With this option the
-                gsutil exit status will be 0 even if some ACLs couldn't be
-                set.
+                it to continue when it encounters errors. If some of the ACLs
+                couldn't be set, gsutil's exit status will be non-zero even if
+                this flag is set. This option is implicitly set when running
+                "gsutil -m acl...".
 """
 
 _CH_DESCRIPTION = """
@@ -132,7 +130,7 @@
   deleting one grant and adding a different grant, the ACLs being updated will
   never be left in an intermediate state where one grant has been deleted but
   the second grant not yet added. Each change specifies a user or group grant
-  to add or delete, and for grant additions, one of R, W, FC (for the
+  to add or delete, and for grant additions, one of R, W, O (for the
   permission to be granted). A more formal description is provided in a later
   section; below we provide examples.
 
@@ -144,10 +142,10 @@
 
     gsutil acl ch -u john.doe@example.com:WRITE gs://example-bucket
 
-  Grant the group admins@example.com FULL_CONTROL access to all jpg files in
+  Grant the group admins@example.com OWNER access to all jpg files in
   the top level of example-bucket:
 
-    gsutil acl ch -g admins@example.com:FC gs://example-bucket/*.jpg
+    gsutil acl ch -g admins@example.com:O gs://example-bucket/*.jpg
 
   Grant the user with the specified canonical ID READ access to all objects
   in example-bucket that begin with folder/:
@@ -156,6 +154,11 @@
       -u 84fac329bceSAMPLE777d5d22b8SAMPLE785ac2SAMPLE2dfcf7c4adf34da46:R \\
       gs://example-bucket/folder/
 
+  Grant the service account foo@developer.gserviceaccount.com WRITE access to
+  the bucket example-bucket:
+
+    gsutil acl ch -u foo@developer.gserviceaccount.com:W gs://example-bucket
+
   Grant all users from my-domain.org READ access to the bucket
   gcs.my-domain.org:
 
@@ -168,62 +171,64 @@
 
   If you have a large number of objects to update, enabling multi-threading
   with the gsutil -m flag can significantly improve performance. The
-  following command adds FULL_CONTROL for admin@example.org using
+  following command adds OWNER for admin@example.org using
   multi-threading:
 
-    gsutil -m acl ch -R -u admin@example.org:FC gs://example-bucket
+    gsutil -m acl ch -R -u admin@example.org:O gs://example-bucket
 
   Grant READ access to everyone from my-domain.org and to all authenticated
-  users, and grant FULL_CONTROL to admin@mydomain.org, for the buckets
+  users, and grant OWNER to admin@mydomain.org, for the buckets
   my-bucket and my-other-bucket, with multi-threading enabled:
 
     gsutil -m acl ch -R -g my-domain.org:R -g AllAuth:R \\
-      -u admin@mydomain.org:FC gs://my-bucket/ gs://my-other-bucket
+      -u admin@mydomain.org:O gs://my-bucket/ gs://my-other-bucket
 
-<B>CH PERMISSIONS</B>
-  You may specify the following permissions with either their shorthand or
+<B>CH ROLES</B>
+  You may specify the following roles with either their shorthand or
   their full name:
 
     R: READ
     W: WRITE
-    FC: FULL_CONTROL
+    O: OWNER
 
-<B>CH SCOPES</B>
-  There are four different scopes: Users, Groups, All Authenticated Users,
+<B>CH ENTITIES</B>
+  There are four different entity types: Users, Groups, All Authenticated Users,
   and All Users.
 
   Users are added with -u and a plain ID or email address, as in
-  "-u john-doe@gmail.com:r"
+  "-u john-doe@gmail.com:r". Note: Service Accounts are considered to be users.
 
   Groups are like users, but specified with the -g flag, as in
   "-g power-users@example.com:fc". Groups may also be specified as a full
   domain, as in "-g my-company.com:r".
 
   AllAuthenticatedUsers and AllUsers are specified directly, as
-  in "-g AllUsers:R" or "-g AllAuthenticatedUsers:FC". These are case
+  in "-g AllUsers:R" or "-g AllAuthenticatedUsers:O". These are case
   insensitive, and may be shortened to "all" and "allauth", respectively.
 
-  Removing permissions is specified with the -d flag and an ID, email
+  Removing roles is specified with the -d flag and an ID, email
   address, domain, or one of AllUsers or AllAuthenticatedUsers.
 
-  Many scopes can be specified on the same command line, allowing bundled
-  changes to be executed in a single run. This will reduce the number of
+  Many entities' roles can be specified on the same command line, allowing
+  bundled changes to be executed in a single run. This will reduce the number of
   requests made to the server.
 
 <B>CH OPTIONS</B>
   The "ch" sub-command has the following options
 
     -R, -r      Performs acl ch request recursively, to all objects under the
-                specified URI.
+                specified URL.
 
-    -u          Add or modify a user permission as specified in the SCOPES
-                and PERMISSIONS sections.
+    -u          Add or modify a user entity's role.
 
-    -g          Add or modify a group permission as specified in the SCOPES
-                and PERMISSIONS sections.
+    -g          Add or modify a group entity's role.
 
-    -d          Remove all permissions associated with the matching argument,
-                as specified in the SCOPES and PERMISSIONS sections
+    -d          Remove all roles associated with the matching entity.
+
+    -f          Normally gsutil stops at the first error. The -f option causes
+                it to continue when it encounters errors. With this option the
+                gsutil exit status will be 0 even if some ACLs couldn't be
+                changed.
 """
 
 _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') +
@@ -233,60 +238,50 @@
   The acl command has three sub-commands:
 """ + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION, _CH_DESCRIPTION]))
 
-_detailed_help_text = CreateHelpText(_SYNOPSIS, _DESCRIPTION)
+_DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION)
 
 _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION)
 _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION)
 _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION)
 
+
 def _ApplyExceptionHandler(cls, exception):
-  cls.logger.error('Encountered a problem: {0}'.format(exception))
+  cls.logger.error('Encountered a problem: %s', exception)
   cls.everything_set_okay = False
 
-def _ApplyAclChangesWrapper(cls, uri_or_expansion_result):
-  cls.ApplyAclChanges(uri_or_expansion_result)
 
+def _ApplyAclChangesWrapper(cls, url_or_expansion_result, thread_state=None):
+  cls.ApplyAclChanges(url_or_expansion_result, thread_state=thread_state)
 
+
 class AclCommand(Command):
   """Implementation of gsutil acl command."""
 
-  # Command specification (processed by parent class).
-  command_spec = {
-    # Name of command.
-    COMMAND_NAME : 'acl',
-    # List of command name aliases.
-    COMMAND_NAME_ALIASES : ['getacl', 'setacl', 'chacl'],
-    # Min number of args required by this command.
-    MIN_ARGS : 2,
-    # Max number of args required by this command, or NO_MAX.
-    MAX_ARGS : NO_MAX,
-    # Getopt-style string specifying acceptable sub args.
-    SUPPORTED_SUB_ARGS : 'afRrvg:u:d:',
-    # True if file URIs acceptable for this command.
-    FILE_URIS_OK : False,
-    # True if provider-only URIs acceptable for this command.
-    PROVIDER_URIS_OK : False,
-    # Index in args of first URI arg.
-    URIS_START_ARG : 1,
-  }
-  help_spec = {
-    # Name of command or auxiliary help info for which this help applies.
-    HELP_NAME : 'acl',
-    # List of help name aliases.
-    HELP_NAME_ALIASES : ['getacl', 'setacl', 'chmod', 'chacl'],
-    # Type of help:
-    HELP_TYPE : HelpType.COMMAND_HELP,
-    # One line summary of this help.
-    HELP_ONE_LINE_SUMMARY : 'Get, set, or change bucket and/or object ACLs',
-    # The full help text.
-    HELP_TEXT : _detailed_help_text,
-    # Help text for sub-commands.
-    SUBCOMMAND_HELP_TEXT : {'get' : _get_help_text,
-                            'set' : _set_help_text,
-                            'ch' : _ch_help_text},
-  }
+  # Command specification. See base class for documentation.
+  command_spec = Command.CreateCommandSpec(
+      'acl',
+      command_name_aliases=['getacl', 'setacl', 'chacl'],
+      min_args=2,
+      max_args=NO_MAX,
+      supported_sub_args='afRrg:u:d:',
+      file_url_ok=False,
+      provider_url_ok=False,
+      urls_start_arg=1,
+      gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
+      gs_default_api=ApiSelector.JSON,
+  )
+  # Help specification. See help_provider.py for documentation.
+  help_spec = Command.HelpSpec(
+      help_name='acl',
+      help_name_aliases=['getacl', 'setacl', 'chmod', 'chacl'],
+      help_type='command_help',
+      help_one_line_summary='Get, set, or change bucket and/or object ACLs',
+      help_text=_DETAILED_HELP_TEXT,
+      subcommand_help_text={
+          'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text},
+  )
 
-  def _CalculateUrisStartArg(self):
+  def _CalculateUrlsStartArg(self):
     if not self.args:
       self._RaiseWrongNumberOfArgumentsException()
     if (self.args[0].lower() == 'set') or (self.command_alias_used == 'setacl'):
@@ -295,6 +290,7 @@
       return 0
 
   def _SetAcl(self):
+    """Parses options and sets ACLs on the specified buckets/objects."""
     self.continue_on_error = False
     if self.sub_opts:
       for o, unused_a in self.sub_opts:
@@ -304,32 +300,24 @@
           self.continue_on_error = True
         elif o == '-r' or o == '-R':
           self.recursion_requested = True
-        elif o == '-v':
-          self.logger.warning('WARNING: The %s -v option is no longer'
-                              ' needed, and will eventually be '
-                              'removed.\n' % self.command_name)
     try:
-      self.SetAclCommandHelper()
-    except GSResponseError as e:
-      if e.code == 'AccessDenied' and e.reason == 'Forbidden' \
-          and e.status == 403:
-        self._WarnServiceAccounts()
+      self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler)
+    except AccessDeniedException, unused_e:
+      self._WarnServiceAccounts()
       raise
+    if not self.everything_set_okay:
+      raise CommandException('ACLs for some objects could not be set.')
 
-  def _GetAcl(self):
-    try:
-      self.GetAclCommandHelper()
-    except GSResponseError as e:
-      if e.code == 'AccessDenied' and e.reason == 'Forbidden' \
-          and e.status == 403:
-        self._WarnServiceAccounts()
-      raise
-    
   def _ChAcl(self):
+    """Parses options and changes ACLs on the specified buckets/objects."""
+    self.parse_versions = True
     self.changes = []
+    self.continue_on_error = False
 
     if self.sub_opts:
       for o, a in self.sub_opts:
+        if o == '-f':
+          self.continue_on_error = True
         if o == '-g':
           self.changes.append(
               aclhelpers.AclChange(a, scope_type=aclhelpers.ChangeType.GROUP))
@@ -346,108 +334,94 @@
           'Please specify at least one access change '
           'with the -g, -u, or -d flags')
 
-    storage_uri = self.UrisAreForSingleProvider(self.args)
-    if not (storage_uri and storage_uri.get_provider().name == 'google'):
+    if (not UrlsAreForSingleProvider(self.args) or
+        StorageUrlFromString(self.args[0]).scheme != 'gs'):
       raise CommandException(
-          'The "{0}" command can only be used with gs:// URIs'.format(
+          'The "{0}" command can only be used with gs:// URLs'.format(
               self.command_name))
 
-    bulk_uris = set()
-    for uri_arg in self.args:
-      for result in self.WildcardIterator(uri_arg):
-        uri = result.uri
-        if uri.names_bucket():
-          if self.recursion_requested:
-            bulk_uris.add(uri.clone_replace_name('*').uri)
-          else:
-            # If applying to a bucket directly, the threading machinery will
-            # break, so we have to apply now, in the main thread.
-            self.ApplyAclChanges(uri)
-        else:
-          bulk_uris.add(uri_arg)
-
-    try:
-      name_expansion_iterator = name_expansion.NameExpansionIterator(
-          self.command_name, self.proj_id_handler, self.headers, self.debug,
-          self.logger, self.bucket_storage_uri_class, bulk_uris,
-          self.recursion_requested)
-    except CommandException as e:
-      # NameExpansionIterator will complain if there are no URIs, but we don't
-      # want to throw an error if we handled bucket URIs.
-      if e.reason == 'No URIs matched':
-        return 0
-      else:
-        raise e
-
     self.everything_set_okay = True
-    self.Apply(_ApplyAclChangesWrapper,
-               name_expansion_iterator,
-               _ApplyExceptionHandler)
+    self.ApplyAclFunc(_ApplyAclChangesWrapper, _ApplyExceptionHandler,
+                      self.args)
     if not self.everything_set_okay:
       raise CommandException('ACLs for some objects could not be set.')
 
-  @Retry(GSResponseError, tries=3, timeout_secs=1)
-  def ApplyAclChanges(self, uri_or_expansion_result):
-    """Applies the changes in self.changes to the provided URI."""
-    if isinstance(uri_or_expansion_result, name_expansion.NameExpansionResult):
-      uri = self.suri_builder.StorageUri(
-          uri_or_expansion_result.expanded_uri_str)
+  @Retry(ServiceException, tries=3, timeout_secs=1)
+  def ApplyAclChanges(self, name_expansion_result, thread_state=None):
+    """Applies the changes in self.changes to the provided URL.
+
+    Args:
+      name_expansion_result: NameExpansionResult describing the target object.
+      thread_state: If present, gsutil Cloud API instance to apply the changes.
+    """
+    if thread_state:
+      gsutil_api = thread_state
     else:
-      uri = uri_or_expansion_result
+      gsutil_api = self.gsutil_api
 
-    try:
-      current_acl = uri.get_acl()
-    except GSResponseError as e:
-      if (e.code == 'AccessDenied' and e.reason == 'Forbidden'
-          and e.status == 403):
-        self._WarnServiceAccounts()
-      self.logger.warning('Failed to set acl for {0}: {1}'
-                          .format(uri, e.reason))
+    url = name_expansion_result.expanded_storage_url
+
+    if url.IsBucket():
+      bucket = gsutil_api.GetBucket(url.bucket_name, provider=url.scheme,
+                                    fields=['acl', 'metageneration'])
+      current_acl = bucket.acl
+    elif url.IsObject():
+      gcs_object = gsutil_api.GetObjectMetadata(
+          url.bucket_name, url.object_name, provider=url.scheme,
+          generation=url.generation,
+          fields=['acl', 'generation', 'metageneration'])
+      current_acl = gcs_object.acl
+    if not current_acl:
+      self._WarnServiceAccounts()
+      self.logger.warning('Failed to set acl for %s. Please ensure you have '
+                          'OWNER-role access to this resource.', url)
       return
 
     modification_count = 0
     for change in self.changes:
-      modification_count += change.Execute(uri, current_acl, self.logger)
+      modification_count += change.Execute(url, current_acl, 'acl', self.logger)
     if modification_count == 0:
-      self.logger.info('No changes to {0}'.format(uri))
+      self.logger.info('No changes to %s', url)
       return
 
-    # TODO: Remove the concept of forcing when boto provides access to
-    # bucket generation and metageneration.
-    headers = dict(self.headers)
-    force = uri.names_bucket()
-    if not force:
-      key = uri.get_key()
-      headers['x-goog-if-generation-match'] = key.generation
-      headers['x-goog-if-metageneration-match'] = key.metageneration
-
-    # If this fails because of a precondition, it will raise a
-    # GSResponseError for @Retry to handle.
     try:
-      uri.set_acl(current_acl, uri.object_name, False, headers)
-    except GSResponseError as e:
+      if url.IsBucket():
+        preconditions = Preconditions(meta_gen_match=bucket.metageneration)
+        bucket_metadata = apitools_messages.Bucket(acl=current_acl)
+        gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
+                               preconditions=preconditions,
+                               provider=url.scheme, fields=['id'])
+      else:  # Object
+        preconditions = Preconditions(gen_match=gcs_object.generation,
+                                      meta_gen_match=gcs_object.metageneration)
+
+        object_metadata = apitools_messages.Object(acl=current_acl)
+        gsutil_api.PatchObjectMetadata(
+            url.bucket_name, url.object_name, object_metadata,
+            preconditions=preconditions, provider=url.scheme,
+            generation=url.generation)
+    except BadRequestException as e:
       # Don't retry on bad requests, e.g. invalid email address.
-      if getattr(e, 'status', None) == 400:
-        raise CommandException('Received bad request from server: %s' % str(e))
-      raise
-    self.logger.info('Updated ACL on {0}'.format(uri))
+      raise CommandException('Received bad request from server: %s' % str(e))
 
-  # Command entry point.
+    self.logger.info('Updated ACL on %s', url)
+
   def RunCommand(self):
+    """Command entry point for the acl command."""
     action_subcommand = self.args.pop(0)
-    (self.sub_opts, self.args) = getopt.getopt(self.args,
-          self.command_spec[SUPPORTED_SUB_ARGS])
+    self.sub_opts, self.args = getopt.getopt(
+        self.args, self.command_spec.supported_sub_args)
     self.CheckArguments()
+    self.def_acl = False
     if action_subcommand == 'get':
-      func = self._GetAcl
+      self.GetAndPrintAcl(self.args[0])
     elif action_subcommand == 'set':
-      func = self._SetAcl
+      self._SetAcl()
     elif action_subcommand in ('ch', 'change'):
-      func = self._ChAcl
+      self._ChAcl()
     else:
       raise CommandException(('Invalid subcommand "%s" for the %s command.\n'
-                             'See "gsutil help acl".') %
+                              'See "gsutil help acl".') %
                              (action_subcommand, self.command_name))
 
-    func()
     return 0