Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1075)

Side by Side Diff: third_party/gsutil/gslib/commands/rm.py

Issue 12042069: Scripts to download files from google storage based on sha1 sums (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/depot_tools.git@master
Patch Set: Removed gsutil/tests and gsutil/docs Created 7 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 # Copyright 2011 Google Inc.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import boto
16
17 from boto.exception import GSResponseError
18 from gslib.command import Command
19 from gslib.command import COMMAND_NAME
20 from gslib.command import COMMAND_NAME_ALIASES
21 from gslib.command import CONFIG_REQUIRED
22 from gslib.command import FILE_URIS_OK
23 from gslib.command import MAX_ARGS
24 from gslib.command import MIN_ARGS
25 from gslib.command import PROVIDER_URIS_OK
26 from gslib.command import SUPPORTED_SUB_ARGS
27 from gslib.command import URIS_START_ARG
28 from gslib.exception import CommandException
29 from gslib.help_provider import HELP_NAME
30 from gslib.help_provider import HELP_NAME_ALIASES
31 from gslib.help_provider import HELP_ONE_LINE_SUMMARY
32 from gslib.help_provider import HELP_TEXT
33 from gslib.help_provider import HelpType
34 from gslib.help_provider import HELP_TYPE
35 from gslib.name_expansion import NameExpansionIterator
36 from gslib.util import NO_MAX
37
38 _detailed_help_text = ("""
39 <B>SYNOPSIS</B>
40 gsutil rm [-f] [-R] uri...
41
42
43 <B>DESCRIPTION</B>
44 The gsutil rm command removes objects.
45 For example, the command:
46
47 gsutil rm gs://bucket/subdir/*
48
49 will remove all objects in gs://bucket/subdir, but not in any of its
50 sub-directories. In contrast:
51
52 gsutil rm gs://bucket/subdir/**
53
54 will remove all objects under gs://bucket/subdir or any of its
55 subdirectories.
56
57 You can also use the -R option to specify recursive object deletion. Thus, for
58 example, the following two commands will both remove all objects in a bucket:
59
60 gsutil rm gs://bucket/**
61 gsutil rm -R gs://bucket
62
63 If you have a large number of objects to remove you might want to use the
64 gsutil -m option, to perform a parallel (multi-threaded/multi-processing)
65 removes:
66
67 gsutil -m rm -R gs://my_bucket/subdir
68
69 Note that gsutil rm will refuse to remove files from the local
70 file system. For example this will fail:
71
72 gsutil rm *.txt
73
74
75 <B>OPTIONS</B>
76 -f Continues silently (without printing error messages) despite
77 errors when removing multiple objects.
78
79 -R, -r Causes bucket contents to be removed recursively (i.e., including
80 all objects and subdirectories). Will not delete the bucket
81 itself; you need to run the gsutil rb command separately to do
82 that.
83
84 -a Delete all versions of an object.
85
86 -v Parses uris for version / generation numbers (only applicable in
87 version-enabled buckets). For example:
88
89 gsutil rm -v gs://bucket/object#1348772910166013.1
90
91 Note that wildcards are not permitted while using this flag.
92 """)
93
94
95 class RmCommand(Command):
96 """Implementation of gsutil rm command."""
97
98 # Command specification (processed by parent class).
99 command_spec = {
100 # Name of command.
101 COMMAND_NAME : 'rm',
102 # List of command name aliases.
103 COMMAND_NAME_ALIASES : ['del', 'delete', 'remove'],
104 # Min number of args required by this command.
105 MIN_ARGS : 1,
106 # Max number of args required by this command, or NO_MAX.
107 MAX_ARGS : NO_MAX,
108 # Getopt-style string specifying acceptable sub args.
109 SUPPORTED_SUB_ARGS : 'afrRv',
110 # True if file URIs acceptable for this command.
111 FILE_URIS_OK : False,
112 # True if provider-only URIs acceptable for this command.
113 PROVIDER_URIS_OK : False,
114 # Index in args of first URI arg.
115 URIS_START_ARG : 0,
116 # True if must configure gsutil before running command.
117 CONFIG_REQUIRED : True,
118 }
119 help_spec = {
120 # Name of command or auxiliary help info for which this help applies.
121 HELP_NAME : 'rm',
122 # List of help name aliases.
123 HELP_NAME_ALIASES : ['del', 'delete', 'remove'],
124 # Type of help:
125 HELP_TYPE : HelpType.COMMAND_HELP,
126 # One line summary of this help.
127 HELP_ONE_LINE_SUMMARY : 'Remove objects',
128 # The full help text.
129 HELP_TEXT : _detailed_help_text,
130 }
131
132 # Command entry point.
133 def RunCommand(self):
134 # self.recursion_requested initialized in command.py (so can be checked
135 # in parent class for all commands).
136 delete_all_versions = False
137 continue_on_error = False
138 parse_versions = False
139 if self.sub_opts:
140 for o, unused_a in self.sub_opts:
141 if o == '-a':
142 delete_all_versions = True
143 elif o == '-f':
144 continue_on_error = True
145 elif o == '-r' or o == '-R':
146 self.recursion_requested = True
147 elif o == '-v':
148 parse_versions = True
149
150 # Used to track if any files failed to be removed.
151 self.everything_removed_okay = True
152
153 if parse_versions and delete_all_versions:
154 raise CommandException(
155 '"rm" does not permit "-a" and "-v" commands simultaneously. If you '
156 'wish to delete only one object version, use "-v". Use "-a" to '
157 'delete all versions.')
158
159 def _RemoveExceptionHandler(e):
160 """Simple exception handler to allow post-completion status."""
161 self.THREADED_LOGGER.error(str(e))
162 self.everything_removed_okay = False
163
164 def _RemoveFunc(name_expansion_result):
165 exp_src_uri = self.suri_builder.StorageUri(
166 name_expansion_result.GetExpandedUriStr(),
167 parse_version=name_expansion_result.parse_version)
168 if exp_src_uri.names_container():
169 if exp_src_uri.is_cloud_uri():
170 # Before offering advice about how to do rm + rb, ensure those
171 # commands won't fail because of bucket naming problems.
172 boto.s3.connection.check_lowercase_bucketname(exp_src_uri.bucket_name)
173 uri_str = exp_src_uri.object_name.rstrip('/')
174 raise CommandException('"rm" command will not remove buckets. To '
175 'delete this/these bucket(s) do:\n\tgsutil rm '
176 '%s/*\n\tgsutil rb %s' % (uri_str, uri_str))
177
178 # In versioned buckets, current versions need to be deleted twice - the
179 # first delete just marks the object deleted without removing data.
180 if delete_all_versions and name_expansion_result.is_current_version:
181 self.THREADED_LOGGER.info('Removing current version %s...',
182 name_expansion_result.expanded_uri_str)
183 try:
184 exp_src_uri.delete_key(validate=False, headers=self.headers)
185 except:
186 if continue_on_error:
187 self.everything_removed_okay = False
188 else:
189 raise
190
191 # Perform delete.
192 self.THREADED_LOGGER.info('Removing %s...',
193 name_expansion_result.expanded_uri_str)
194 try:
195 exp_src_uri.delete_key(validate=False, headers=self.headers)
196 except:
197 if continue_on_error:
198 self.everything_removed_okay = False
199 else:
200 raise
201
202 # Expand wildcards, dirs, buckets, and bucket subdirs in URIs.
203 name_expansion_iterator = NameExpansionIterator(
204 self.command_name, self.proj_id_handler, self.headers, self.debug,
205 self.bucket_storage_uri_class, self.args, self.recursion_requested,
206 flat=self.recursion_requested, all_versions=delete_all_versions,
207 for_all_version_delete=delete_all_versions,
208 parse_versions=parse_versions)
209
210 # Perform remove requests in parallel (-m) mode, if requested, using
211 # configured number of parallel processes and threads. Otherwise,
212 # perform requests with sequential function calls in current process.
213 self.Apply(_RemoveFunc, name_expansion_iterator, _RemoveExceptionHandler)
214
215 if not self.everything_removed_okay:
216 raise CommandException('Some files could not be removed.')
217
218 return 0
219
220 # Test specification. See definition of test_steps in base class for
221 # details on how to populate these fields.
222 num_test_buckets = 1
223 test_steps = [
224 ('stage empty file', 'echo -n \'\' > $F9', 0, None),
225 ('enable versioning', 'gsutil setversioning on gs://$B0', 0, None),
226 ('upload initial version', 'echo \'data1\' | gsutil cp - gs://$B0/$O0', 0,
227 None),
228 ('upload new version', 'echo \'data2\' | gsutil cp - gs://$B0/$O0', 0,
229 None),
230 # Test that "rm -a" for an object with a current version works.
231 ('delete all versions', 'gsutil -m rm -a gs://$B0/$O0', 0, None),
232 ('check all versions gone', 'gsutil ls -a gs://$B0/ > $F1', 0,
233 ('$F1', '$F9')),
234 ('upload initial version', 'echo \'data1\' | gsutil cp - gs://$B0/$O0', 0,
235 None),
236 ('upload new version', 'echo \'data2\' | gsutil cp - gs://$B0/$O0', 0,
237 None),
238 ('delete current version', 'gsutil rm gs://$B0/$O0', 0, None),
239 # Test that "rm -a" for an object without a current version works.
240 ('delete all versions', 'gsutil -m rm -a gs://$B0/$O0', 0, None),
241 ('check all versions gone', 'gsutil ls -a gs://$B0/ > $F1', 0,
242 ('$F1', '$F9')),
243 ('rm -a fails for missing obj', 'gsutil rm -a gs://$B0/$O0', 1, None),
244 ]
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698