OLD | NEW |
(Empty) | |
| 1 # Copyright 2011 Google Inc. |
| 2 # |
| 3 # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 # you may not use this file except in compliance with the License. |
| 5 # You may obtain a copy of the License at |
| 6 # |
| 7 # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 # |
| 9 # Unless required by applicable law or agreed to in writing, software |
| 10 # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 # See the License for the specific language governing permissions and |
| 13 # limitations under the License. |
| 14 |
| 15 import boto |
| 16 |
| 17 from boto.exception import GSResponseError |
| 18 from gslib.command import Command |
| 19 from gslib.command import COMMAND_NAME |
| 20 from gslib.command import COMMAND_NAME_ALIASES |
| 21 from gslib.command import CONFIG_REQUIRED |
| 22 from gslib.command import FILE_URIS_OK |
| 23 from gslib.command import MAX_ARGS |
| 24 from gslib.command import MIN_ARGS |
| 25 from gslib.command import PROVIDER_URIS_OK |
| 26 from gslib.command import SUPPORTED_SUB_ARGS |
| 27 from gslib.command import URIS_START_ARG |
| 28 from gslib.exception import CommandException |
| 29 from gslib.help_provider import HELP_NAME |
| 30 from gslib.help_provider import HELP_NAME_ALIASES |
| 31 from gslib.help_provider import HELP_ONE_LINE_SUMMARY |
| 32 from gslib.help_provider import HELP_TEXT |
| 33 from gslib.help_provider import HelpType |
| 34 from gslib.help_provider import HELP_TYPE |
| 35 from gslib.name_expansion import NameExpansionIterator |
| 36 from gslib.util import NO_MAX |
| 37 |
| 38 _detailed_help_text = (""" |
| 39 <B>SYNOPSIS</B> |
| 40 gsutil rm [-f] [-R] uri... |
| 41 |
| 42 |
| 43 <B>DESCRIPTION</B> |
| 44 The gsutil rm command removes objects. |
| 45 For example, the command: |
| 46 |
| 47 gsutil rm gs://bucket/subdir/* |
| 48 |
| 49 will remove all objects in gs://bucket/subdir, but not in any of its |
| 50 sub-directories. In contrast: |
| 51 |
| 52 gsutil rm gs://bucket/subdir/** |
| 53 |
| 54 will remove all objects under gs://bucket/subdir or any of its |
| 55 subdirectories. |
| 56 |
| 57 You can also use the -R option to specify recursive object deletion. Thus, for |
| 58 example, the following two commands will both remove all objects in a bucket: |
| 59 |
| 60 gsutil rm gs://bucket/** |
| 61 gsutil rm -R gs://bucket |
| 62 |
| 63 If you have a large number of objects to remove you might want to use the |
| 64 gsutil -m option, to perform a parallel (multi-threaded/multi-processing) |
| 65 removes: |
| 66 |
| 67 gsutil -m rm -R gs://my_bucket/subdir |
| 68 |
| 69 Note that gsutil rm will refuse to remove files from the local |
| 70 file system. For example this will fail: |
| 71 |
| 72 gsutil rm *.txt |
| 73 |
| 74 |
| 75 <B>OPTIONS</B> |
| 76 -f Continues silently (without printing error messages) despite |
| 77 errors when removing multiple objects. |
| 78 |
| 79 -R, -r Causes bucket contents to be removed recursively (i.e., including |
| 80 all objects and subdirectories). Will not delete the bucket |
| 81 itself; you need to run the gsutil rb command separately to do |
| 82 that. |
| 83 |
| 84 -a Delete all versions of an object. |
| 85 |
| 86 -v Parses uris for version / generation numbers (only applicable in |
| 87 version-enabled buckets). For example: |
| 88 |
| 89 gsutil rm -v gs://bucket/object#1348772910166013.1 |
| 90 |
| 91 Note that wildcards are not permitted while using this flag. |
| 92 """) |
| 93 |
| 94 |
| 95 class RmCommand(Command): |
| 96 """Implementation of gsutil rm command.""" |
| 97 |
| 98 # Command specification (processed by parent class). |
| 99 command_spec = { |
| 100 # Name of command. |
| 101 COMMAND_NAME : 'rm', |
| 102 # List of command name aliases. |
| 103 COMMAND_NAME_ALIASES : ['del', 'delete', 'remove'], |
| 104 # Min number of args required by this command. |
| 105 MIN_ARGS : 1, |
| 106 # Max number of args required by this command, or NO_MAX. |
| 107 MAX_ARGS : NO_MAX, |
| 108 # Getopt-style string specifying acceptable sub args. |
| 109 SUPPORTED_SUB_ARGS : 'afrRv', |
| 110 # True if file URIs acceptable for this command. |
| 111 FILE_URIS_OK : False, |
| 112 # True if provider-only URIs acceptable for this command. |
| 113 PROVIDER_URIS_OK : False, |
| 114 # Index in args of first URI arg. |
| 115 URIS_START_ARG : 0, |
| 116 # True if must configure gsutil before running command. |
| 117 CONFIG_REQUIRED : True, |
| 118 } |
| 119 help_spec = { |
| 120 # Name of command or auxiliary help info for which this help applies. |
| 121 HELP_NAME : 'rm', |
| 122 # List of help name aliases. |
| 123 HELP_NAME_ALIASES : ['del', 'delete', 'remove'], |
| 124 # Type of help: |
| 125 HELP_TYPE : HelpType.COMMAND_HELP, |
| 126 # One line summary of this help. |
| 127 HELP_ONE_LINE_SUMMARY : 'Remove objects', |
| 128 # The full help text. |
| 129 HELP_TEXT : _detailed_help_text, |
| 130 } |
| 131 |
| 132 # Command entry point. |
| 133 def RunCommand(self): |
| 134 # self.recursion_requested initialized in command.py (so can be checked |
| 135 # in parent class for all commands). |
| 136 delete_all_versions = False |
| 137 continue_on_error = False |
| 138 parse_versions = False |
| 139 if self.sub_opts: |
| 140 for o, unused_a in self.sub_opts: |
| 141 if o == '-a': |
| 142 delete_all_versions = True |
| 143 elif o == '-f': |
| 144 continue_on_error = True |
| 145 elif o == '-r' or o == '-R': |
| 146 self.recursion_requested = True |
| 147 elif o == '-v': |
| 148 parse_versions = True |
| 149 |
| 150 # Used to track if any files failed to be removed. |
| 151 self.everything_removed_okay = True |
| 152 |
| 153 if parse_versions and delete_all_versions: |
| 154 raise CommandException( |
| 155 '"rm" does not permit "-a" and "-v" commands simultaneously. If you ' |
| 156 'wish to delete only one object version, use "-v". Use "-a" to ' |
| 157 'delete all versions.') |
| 158 |
| 159 def _RemoveExceptionHandler(e): |
| 160 """Simple exception handler to allow post-completion status.""" |
| 161 self.THREADED_LOGGER.error(str(e)) |
| 162 self.everything_removed_okay = False |
| 163 |
| 164 def _RemoveFunc(name_expansion_result): |
| 165 exp_src_uri = self.suri_builder.StorageUri( |
| 166 name_expansion_result.GetExpandedUriStr(), |
| 167 parse_version=name_expansion_result.parse_version) |
| 168 if exp_src_uri.names_container(): |
| 169 if exp_src_uri.is_cloud_uri(): |
| 170 # Before offering advice about how to do rm + rb, ensure those |
| 171 # commands won't fail because of bucket naming problems. |
| 172 boto.s3.connection.check_lowercase_bucketname(exp_src_uri.bucket_name) |
| 173 uri_str = exp_src_uri.object_name.rstrip('/') |
| 174 raise CommandException('"rm" command will not remove buckets. To ' |
| 175 'delete this/these bucket(s) do:\n\tgsutil rm ' |
| 176 '%s/*\n\tgsutil rb %s' % (uri_str, uri_str)) |
| 177 |
| 178 # In versioned buckets, current versions need to be deleted twice - the |
| 179 # first delete just marks the object deleted without removing data. |
| 180 if delete_all_versions and name_expansion_result.is_current_version: |
| 181 self.THREADED_LOGGER.info('Removing current version %s...', |
| 182 name_expansion_result.expanded_uri_str) |
| 183 try: |
| 184 exp_src_uri.delete_key(validate=False, headers=self.headers) |
| 185 except: |
| 186 if continue_on_error: |
| 187 self.everything_removed_okay = False |
| 188 else: |
| 189 raise |
| 190 |
| 191 # Perform delete. |
| 192 self.THREADED_LOGGER.info('Removing %s...', |
| 193 name_expansion_result.expanded_uri_str) |
| 194 try: |
| 195 exp_src_uri.delete_key(validate=False, headers=self.headers) |
| 196 except: |
| 197 if continue_on_error: |
| 198 self.everything_removed_okay = False |
| 199 else: |
| 200 raise |
| 201 |
| 202 # Expand wildcards, dirs, buckets, and bucket subdirs in URIs. |
| 203 name_expansion_iterator = NameExpansionIterator( |
| 204 self.command_name, self.proj_id_handler, self.headers, self.debug, |
| 205 self.bucket_storage_uri_class, self.args, self.recursion_requested, |
| 206 flat=self.recursion_requested, all_versions=delete_all_versions, |
| 207 for_all_version_delete=delete_all_versions, |
| 208 parse_versions=parse_versions) |
| 209 |
| 210 # Perform remove requests in parallel (-m) mode, if requested, using |
| 211 # configured number of parallel processes and threads. Otherwise, |
| 212 # perform requests with sequential function calls in current process. |
| 213 self.Apply(_RemoveFunc, name_expansion_iterator, _RemoveExceptionHandler) |
| 214 |
| 215 if not self.everything_removed_okay: |
| 216 raise CommandException('Some files could not be removed.') |
| 217 |
| 218 return 0 |
| 219 |
| 220 # Test specification. See definition of test_steps in base class for |
| 221 # details on how to populate these fields. |
| 222 num_test_buckets = 1 |
| 223 test_steps = [ |
| 224 ('stage empty file', 'echo -n \'\' > $F9', 0, None), |
| 225 ('enable versioning', 'gsutil setversioning on gs://$B0', 0, None), |
| 226 ('upload initial version', 'echo \'data1\' | gsutil cp - gs://$B0/$O0', 0, |
| 227 None), |
| 228 ('upload new version', 'echo \'data2\' | gsutil cp - gs://$B0/$O0', 0, |
| 229 None), |
| 230 # Test that "rm -a" for an object with a current version works. |
| 231 ('delete all versions', 'gsutil -m rm -a gs://$B0/$O0', 0, None), |
| 232 ('check all versions gone', 'gsutil ls -a gs://$B0/ > $F1', 0, |
| 233 ('$F1', '$F9')), |
| 234 ('upload initial version', 'echo \'data1\' | gsutil cp - gs://$B0/$O0', 0, |
| 235 None), |
| 236 ('upload new version', 'echo \'data2\' | gsutil cp - gs://$B0/$O0', 0, |
| 237 None), |
| 238 ('delete current version', 'gsutil rm gs://$B0/$O0', 0, None), |
| 239 # Test that "rm -a" for an object without a current version works. |
| 240 ('delete all versions', 'gsutil -m rm -a gs://$B0/$O0', 0, None), |
| 241 ('check all versions gone', 'gsutil ls -a gs://$B0/ > $F1', 0, |
| 242 ('$F1', '$F9')), |
| 243 ('rm -a fails for missing obj', 'gsutil rm -a gs://$B0/$O0', 1, None), |
| 244 ] |
OLD | NEW |