Roll version of gsutil to 4.15.

third_party/gsutil/gslib/commands/rm.py

 # -*- coding: utf-8 -*-
 # Copyright 2011 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Implementation of Unix-like rm command for cloud storage providers."""
 
 from __future__ import absolute_import
 
+from gslib.cloud_api import BucketNotFoundException
 from gslib.cloud_api import NotEmptyException
+from gslib.cloud_api import NotFoundException
 from gslib.cloud_api import ServiceException
 from gslib.command import Command
 from gslib.command import GetFailureCount
 from gslib.command import ResetFailureCount
 from gslib.command_argument import CommandArgument
 from gslib.cs_api_map import ApiSelector
 from gslib.exception import CommandException
 from gslib.name_expansion import NameExpansionIterator
 from gslib.storage_url import StorageUrlFromString
 from gslib.translation_helper import PreconditionsFromHeaders
@@ skipping 102 matching lines @@
               flag implies the -a flag and will delete all object versions.
 
   -a          Delete all versions of an object.
 """)
 
 
 def _RemoveExceptionHandler(cls, e):
   """Simple exception handler to allow post-completion status."""
   if not cls.continue_on_error:
     cls.logger.error(str(e))
-  cls.everything_removed_okay = False
+  # TODO: Use shared state to track missing bucket names when we get a
+  # BucketNotFoundException. Then improve bucket removal logic and exception
+  # messages.
+  if isinstance(e, BucketNotFoundException):
+    cls.bucket_not_found_count += 1
+    cls.logger.error(str(e))
+  else:
+    cls.op_failure_count += 1
 
 
 # pylint: disable=unused-argument
 def _RemoveFoldersExceptionHandler(cls, e):
   """When removing folders, we don't mind if none exist."""
   if (isinstance(e, CommandException.__class__) and
-      'No URLs matched' in e.message):
+      'No URLs matched' in e.message) or isinstance(e, NotFoundException):
     pass
   else:
     raise e
 
 
 def _RemoveFuncWrapper(cls, name_expansion_result, thread_state=None):
   cls.RemoveFunc(name_expansion_result, thread_state=thread_state)
 
 
 class RmCommand(Command):
@@ skipping 23 matching lines @@
       help_type='command_help',
       help_one_line_summary='Remove objects',
       help_text=_DETAILED_HELP_TEXT,
       subcommand_help_text={},
   )
 
   def RunCommand(self):
     """Command entry point for the rm command."""
     # self.recursion_requested is initialized in command.py (so it can be
     # checked in parent class for all commands).
-    self.continue_on_error = False
+    self.continue_on_error = self.parallel_operations
     self.read_args_from_stdin = False
     self.all_versions = False
     if self.sub_opts:
       for o, unused_a in self.sub_opts:
         if o == '-a':
           self.all_versions = True
         elif o == '-f':
           self.continue_on_error = True
         elif o == '-I':
           self.read_args_from_stdin = True
         elif o == '-r' or o == '-R':
           self.recursion_requested = True
           self.all_versions = True
 
     if self.read_args_from_stdin:
       if self.args:
         raise CommandException('No arguments allowed with the -I flag.')
       url_strs = StdinIterator()
     else:
       if not self.args:
         raise CommandException('The rm command (without -I) expects at '
                                'least one URL.')
       url_strs = self.args
 
+    # Tracks if any deletes failed.
+    self.op_failure_count = 0
+
+    # Tracks if any buckets were missing.
+    self.bucket_not_found_count = 0
+
     bucket_urls_to_delete = []
     bucket_strings_to_delete = []
     if self.recursion_requested:
       bucket_fields = ['id']
       for url_str in url_strs:
         url = StorageUrlFromString(url_str)
         if url.IsBucket() or url.IsProvider():
           for blr in self.WildcardIterator(url_str).IterBuckets(
               bucket_fields=bucket_fields):
             bucket_urls_to_delete.append(blr.storage_url)
             bucket_strings_to_delete.append(url_str)
 
     self.preconditions = PreconditionsFromHeaders(self.headers or {})
 
-    # Used to track if any files failed to be removed.
-    self.everything_removed_okay = True
-
     try:
       # Expand wildcards, dirs, buckets, and bucket subdirs in URLs.
       name_expansion_iterator = NameExpansionIterator(
           self.command_name, self.debug, self.logger, self.gsutil_api,
           url_strs, self.recursion_requested, project_id=self.project_id,
           all_versions=self.all_versions,
           continue_on_error=self.continue_on_error or self.parallel_operations)
 
       # Perform remove requests in parallel (-m) mode, if requested, using
       # configured number of parallel processes and threads. Otherwise,
       # perform requests with sequential function calls in current process.
       self.Apply(_RemoveFuncWrapper, name_expansion_iterator,
                  _RemoveExceptionHandler,
-                 fail_on_error=(not self.continue_on_error))
+                 fail_on_error=(not self.continue_on_error),
+                 shared_attrs=['op_failure_count', 'bucket_not_found_count'])
 
     # Assuming the bucket has versioning enabled, url's that don't map to
     # objects should throw an error even with all_versions, since the prior
     # round of deletes only sends objects to a history table.
     # This assumption that rm -a is only called for versioned buckets should be
     # corrected, but the fix is non-trivial.
     except CommandException as e:
       # Don't raise if there are buckets to delete -- it's valid to say:
       #   gsutil rm -r gs://some_bucket
       # if the bucket is empty.
       if not bucket_urls_to_delete and not self.continue_on_error:
         raise
       # Reset the failure count if we failed due to an empty bucket that we're
       # going to delete.
       msg = 'No URLs matched: '
       if msg in str(e):
         parts = str(e).split(msg)
         if len(parts) == 2 and parts[1] in bucket_strings_to_delete:
           ResetFailureCount()
+        else:
+          raise
     except ServiceException, e:
       if not self.continue_on_error:
         raise
 
-    if not self.everything_removed_okay and not self.continue_on_error:
+    if self.bucket_not_found_count:
+      raise CommandException('Encountered non-existent bucket during listing')
+
+    if self.op_failure_count and not self.continue_on_error:
       raise CommandException('Some files could not be removed.')
 
     # If this was a gsutil rm -r command covering any bucket subdirs,
     # remove any dir_$folder$ objects (which are created by various web UI
     # tools to simulate folders).
     if self.recursion_requested:
       had_previous_failures = GetFailureCount() > 0
       folder_object_wildcards = []
       for url_str in url_strs:
         url = StorageUrlFromString(url_str)
@@ skipping 22 matching lines @@
     # Now that all data has been deleted, delete any bucket URLs.
     for url in bucket_urls_to_delete:
       self.logger.info('Removing %s...', url)
 
       @Retry(NotEmptyException, tries=3, timeout_secs=1)
       def BucketDeleteWithRetry():
         self.gsutil_api.DeleteBucket(url.bucket_name, provider=url.scheme)
 
       BucketDeleteWithRetry()
 
+    if self.op_failure_count:
+      plural_str = 's' if self.op_failure_count else ''
+      raise CommandException('%d file%s/object%s could not be removed.' % (
+          self.op_failure_count, plural_str, plural_str))
+
     return 0
 
   def RemoveFunc(self, name_expansion_result, thread_state=None):
     gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)
 
     exp_src_url = name_expansion_result.expanded_storage_url
     self.logger.info('Removing %s...', exp_src_url)
     gsutil_api.DeleteObject(
         exp_src_url.bucket_name, exp_src_url.object_name,
         preconditions=self.preconditions, generation=exp_src_url.generation,
         provider=exp_src_url.scheme)