| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright 2011 Google Inc. All Rights Reserved. | |
| 2 # | |
| 3 # Licensed under the Apache License, Version 2.0 (the "License"); | |
| 4 # you may not use this file except in compliance with the License. | |
| 5 # You may obtain a copy of the License at | |
| 6 # | |
| 7 # http://www.apache.org/licenses/LICENSE-2.0 | |
| 8 # | |
| 9 # Unless required by applicable law or agreed to in writing, software | |
| 10 # distributed under the License is distributed on an "AS IS" BASIS, | |
| 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 12 # See the License for the specific language governing permissions and | |
| 13 # limitations under the License. | |
| 14 | |
| 15 from boto.s3.deletemarker import DeleteMarker | |
| 16 from gslib.bucket_listing_ref import BucketListingRef | |
| 17 from gslib.command import Command | |
| 18 from gslib.command import COMMAND_NAME | |
| 19 from gslib.command import COMMAND_NAME_ALIASES | |
| 20 from gslib.command import CONFIG_REQUIRED | |
| 21 from gslib.command import FILE_URIS_OK | |
| 22 from gslib.command import MAX_ARGS | |
| 23 from gslib.command import MIN_ARGS | |
| 24 from gslib.command import PROVIDER_URIS_OK | |
| 25 from gslib.command import SUPPORTED_SUB_ARGS | |
| 26 from gslib.command import URIS_START_ARG | |
| 27 from gslib.exception import CommandException | |
| 28 from gslib.help_provider import HELP_NAME | |
| 29 from gslib.help_provider import HELP_NAME_ALIASES | |
| 30 from gslib.help_provider import HELP_ONE_LINE_SUMMARY | |
| 31 from gslib.help_provider import HELP_TEXT | |
| 32 from gslib.help_provider import HelpType | |
| 33 from gslib.help_provider import HELP_TYPE | |
| 34 from gslib.plurality_checkable_iterator import PluralityCheckableIterator | |
| 35 from gslib.util import ListingStyle | |
| 36 from gslib.util import MakeHumanReadable | |
| 37 from gslib.util import NO_MAX | |
| 38 from gslib.wildcard_iterator import ContainsWildcard | |
| 39 import boto | |
| 40 | |
| 41 _detailed_help_text = (""" | |
| 42 <B>SYNOPSIS</B> | |
| 43 gsutil ls [-a] [-b] [-l] [-L] [-R] [-p proj_id] uri... | |
| 44 | |
| 45 | |
| 46 <B>LISTING PROVIDERS, BUCKETS, SUBDIRECTORIES, AND OBJECTS</B> | |
| 47 If you run gsutil ls without URIs, it lists all of the Google Cloud Storage | |
| 48 buckets under your default project ID: | |
| 49 | |
| 50 gsutil ls | |
| 51 | |
| 52 (For details about projects, see "gsutil help projects" and also the -p | |
| 53 option in the OPTIONS section below.) | |
| 54 | |
| 55 If you specify one or more provider URIs, gsutil ls will list buckets at | |
| 56 each listed provider: | |
| 57 | |
| 58 gsutil ls gs:// | |
| 59 | |
| 60 If you specify bucket URIs, gsutil ls will list objects at the top level of | |
| 61 each bucket, along with the names of each subdirectory. For example: | |
| 62 | |
| 63 gsutil ls gs://bucket | |
| 64 | |
| 65 might produce output like: | |
| 66 | |
| 67 gs://bucket/obj1.htm | |
| 68 gs://bucket/obj2.htm | |
| 69 gs://bucket/images1/ | |
| 70 gs://bucket/images2/ | |
| 71 | |
| 72 The "/" at the end of the last 2 URIs tells you they are subdirectories, | |
| 73 which you can list using: | |
| 74 | |
| 75 gsutil ls gs://bucket/images* | |
| 76 | |
| 77 If you specify object URIs, gsutil ls will list the specified objects. For | |
| 78 example: | |
| 79 | |
| 80 gsutil ls gs://bucket/*.txt | |
| 81 | |
| 82 will list all files whose name matches the above wildcard at the top level | |
| 83 of the bucket. | |
| 84 | |
| 85 See "gsutil help wildcards" for more details on working with wildcards. | |
| 86 | |
| 87 | |
| 88 <B>DIRECTORY BY DIRECTORY, FLAT, and RECURSIVE LISTINGS</B> | |
| 89 Listing a bucket or subdirectory (as illustrated near the end of the previous | |
| 90 section) only shows the objects and names of subdirectories it contains. You | |
| 91 can list all objects in a bucket by using the -R option. For example: | |
| 92 | |
| 93 gsutil ls -R gs://bucket | |
| 94 | |
| 95 will list the top-level objects and buckets, then the objects and | |
| 96 buckets under gs://bucket/images1, then those under gs://bucket/images2, etc. | |
| 97 | |
| 98 If you want to see all objects in the bucket in one "flat" listing use the | |
| 99 recursive ("**") wildcard, like: | |
| 100 | |
| 101 gsutil ls -R gs://bucket/** | |
| 102 | |
| 103 or, for a flat listing of a subdirectory: | |
| 104 | |
| 105 gsutil ls -R gs://bucket/dir/** | |
| 106 | |
| 107 | |
| 108 <B>LISTING OBJECT DETAILS</B> | |
| 109 If you specify the -l option, gsutil will output additional information | |
| 110 about each matching provider, bucket, subdirectory, or object. For example, | |
| 111 | |
| 112 gsutil ls -l gs://bucket/*.txt | |
| 113 | |
| 114 will print the object size, creation time stamp, and name of each matching | |
| 115 object, along with the total count and sum of sizes of all matching objects: | |
| 116 | |
| 117 2276224 2012-03-02T19:25:17 gs://bucket/obj1 | |
| 118 3914624 2012-03-02T19:30:27 gs://bucket/obj2 | |
| 119 TOTAL: 2 objects, 6190848 bytes (5.9 MB) | |
| 120 | |
| 121 Note that the total listed in parentheses above is in mebibytes (or gibibytes, | |
| 122 tebibytes, etc.), which corresponds to the unit of billing measurement for | |
| 123 Google Cloud Storage. | |
| 124 | |
| 125 You can get a listing of all the objects in the top-level bucket directory | |
| 126 (along with the total count and sum of sizes) using a command like: | |
| 127 | |
| 128 gsutil ls -l gs://bucket | |
| 129 | |
| 130 To print additional detail about objects and buckets use the gsutil ls -L | |
| 131 option. For example: | |
| 132 | |
| 133 gsutil ls -L gs://bucket/obj1 | |
| 134 | |
| 135 will print something like: | |
| 136 | |
| 137 gs://bucket/obj1: | |
| 138 Creation Time: Fri, 02 Mar 2012 19:25:17 GMT | |
| 139 Size: 2276224 | |
| 140 Cache-Control: private, max-age=0 | |
| 141 Content-Type: application/x-executable | |
| 142 ETag: 5ca6796417570a586723b7344afffc81 | |
| 143 ACL: <Owner:00b4903a97163d99003117abe64d292561d2b4074fc90ce5c
0e35ac45f66ad70, <<UserById: 00b4903a97163d99003117abe64d292561d2b4074fc90ce5c0e
35ac45f66ad70>: u'FULL_CONTROL'>> | |
| 144 TOTAL: 1 objects, 2276224 bytes (2.17 MB) | |
| 145 | |
| 146 Note that the -L option is slower and more costly to use than the -l option, | |
| 147 because it makes a bucket listing request followed by a HEAD request for | |
| 148 each individual object (rather than just parsing the information it needs | |
| 149 out of a single bucket listing, the way the -l option does). | |
| 150 | |
| 151 See also "gsutil help getacl" for getting a more readable version of the ACL. | |
| 152 | |
| 153 | |
| 154 <B>LISTING BUCKET DETAILS</B> | |
| 155 If you want to see information about the bucket itself, use the -b | |
| 156 option. For example: | |
| 157 | |
| 158 gsutil ls -L -b gs://bucket | |
| 159 | |
| 160 will print something like: | |
| 161 | |
| 162 gs://bucket/ : | |
| 163 24 objects, 29.83 KB | |
| 164 StorageClass: STANDARD | |
| 165 LocationConstraint: US | |
| 166 Versioning enabled: True | |
| 167 ACL: <Owner:00b4903a9740e42c29800f53bd5a9a62a2f96eb3f64a4313a115df3f
3a776bf7, <<GroupById: 00b4903a9740e42c29800f53bd5a9a62a2f96eb3f64a4313a115df3f3
a776bf7>: u'FULL_CONTROL'>> | |
| 168 Default ACL: <> | |
| 169 TOTAL: 24 objects, 30544 bytes (29.83 KB) | |
| 170 | |
| 171 | |
| 172 <B>OPTIONS</B> | |
| 173 -l Prints long listing (owner, length). | |
| 174 | |
| 175 -L Prints even more detail than -l. This is a separate option because | |
| 176 it makes additional service requests (so, takes longer and adds | |
| 177 requests costs). | |
| 178 | |
| 179 -b Prints info about the bucket when used with a bucket URI. | |
| 180 | |
| 181 -p proj_id Specifies the project ID to use for listing buckets. | |
| 182 | |
| 183 -R, -r Requests a recursive listing. | |
| 184 | |
| 185 -a Includes non-current object versions / generations in the listing | |
| 186 (only useful with a versioning-enabled bucket). If combined with | |
| 187 -l option also prints meta-generation for each listed object. | |
| 188 """) | |
| 189 | |
| 190 | |
| 191 class LsCommand(Command): | |
| 192 """Implementation of gsutil ls command.""" | |
| 193 | |
| 194 # Command specification (processed by parent class). | |
| 195 command_spec = { | |
| 196 # Name of command. | |
| 197 COMMAND_NAME : 'ls', | |
| 198 # List of command name aliases. | |
| 199 COMMAND_NAME_ALIASES : ['dir', 'list'], | |
| 200 # Min number of args required by this command. | |
| 201 MIN_ARGS : 0, | |
| 202 # Max number of args required by this command, or NO_MAX. | |
| 203 MAX_ARGS : NO_MAX, | |
| 204 # Getopt-style string specifying acceptable sub args. | |
| 205 SUPPORTED_SUB_ARGS : 'ablLp:rR', | |
| 206 # True if file URIs acceptable for this command. | |
| 207 FILE_URIS_OK : False, | |
| 208 # True if provider-only URIs acceptable for this command. | |
| 209 PROVIDER_URIS_OK : True, | |
| 210 # Index in args of first URI arg. | |
| 211 URIS_START_ARG : 0, | |
| 212 # True if must configure gsutil before running command. | |
| 213 CONFIG_REQUIRED : True, | |
| 214 } | |
| 215 help_spec = { | |
| 216 # Name of command or auxiliary help info for which this help applies. | |
| 217 HELP_NAME : 'ls', | |
| 218 # List of help name aliases. | |
| 219 HELP_NAME_ALIASES : ['dir', 'list'], | |
| 220 # Type of help: | |
| 221 HELP_TYPE : HelpType.COMMAND_HELP, | |
| 222 # One line summary of this help. | |
| 223 HELP_ONE_LINE_SUMMARY : 'List providers, buckets, or objects', | |
| 224 # The full help text. | |
| 225 HELP_TEXT : _detailed_help_text, | |
| 226 } | |
| 227 | |
| 228 def _PrintBucketInfo(self, bucket_uri, listing_style): | |
| 229 """Print listing info for given bucket. | |
| 230 | |
| 231 Args: | |
| 232 bucket_uri: StorageUri being listed. | |
| 233 listing_style: ListingStyle enum describing type of output desired. | |
| 234 | |
| 235 Returns: | |
| 236 Tuple (total objects, total bytes) in the bucket. | |
| 237 """ | |
| 238 bucket_objs = 0 | |
| 239 bucket_bytes = 0 | |
| 240 if listing_style == ListingStyle.SHORT: | |
| 241 print bucket_uri | |
| 242 else: | |
| 243 for obj in self.WildcardIterator( | |
| 244 bucket_uri.clone_replace_name('**')).IterKeys(): | |
| 245 bucket_objs += 1 | |
| 246 bucket_bytes += obj.size | |
| 247 if listing_style == ListingStyle.LONG: | |
| 248 print '%s : %s objects, %s' % ( | |
| 249 bucket_uri, bucket_objs, MakeHumanReadable(bucket_bytes)) | |
| 250 else: # listing_style == ListingStyle.LONG_LONG: | |
| 251 location_constraint = bucket_uri.get_location(validate=False, | |
| 252 headers=self.headers) | |
| 253 location_output = '' | |
| 254 if location_constraint: | |
| 255 location_output = '\n\tLocationConstraint: %s' % location_constraint | |
| 256 storage_class = bucket_uri.get_storage_class(validate=False, | |
| 257 headers=self.headers) | |
| 258 self.proj_id_handler.FillInProjectHeaderIfNeeded( | |
| 259 'get_acl', bucket_uri, self.headers) | |
| 260 print('%s :\n\t%d objects, %s\n\tStorageClass: %s%s\n' | |
| 261 '\tVersioning enabled: %s\n\tACL: %s\n' | |
| 262 '\tDefault ACL: %s' % ( | |
| 263 bucket_uri, bucket_objs, MakeHumanReadable(bucket_bytes), | |
| 264 storage_class, location_output, | |
| 265 bucket_uri.get_versioning_config(), | |
| 266 bucket_uri.get_acl(False, self.headers), | |
| 267 bucket_uri.get_def_acl(False, self.headers))) | |
| 268 return (bucket_objs, bucket_bytes) | |
| 269 | |
| 270 def _UriStrForObj(self, uri, obj): | |
| 271 """Constructs a URI string for the given object. | |
| 272 | |
| 273 For example if we were iterating gs://*, obj could be an object in one | |
| 274 of the user's buckets enumerated by the ls command. | |
| 275 | |
| 276 Args: | |
| 277 uri: base StorageUri being iterated. | |
| 278 obj: object (Key) being listed. | |
| 279 | |
| 280 Returns: | |
| 281 URI string. | |
| 282 """ | |
| 283 version_info = '' | |
| 284 if self.all_versions: | |
| 285 if uri.get_provider().name == 'google' and obj.generation: | |
| 286 version_info = '#%s' % obj.generation | |
| 287 elif uri.get_provider().name == 'aws' and obj.version_id: | |
| 288 if isinstance(obj, DeleteMarker): | |
| 289 version_info = '#<DeleteMarker>' + str(obj.version_id) | |
| 290 else: | |
| 291 version_info = '#' + str(obj.version_id) | |
| 292 else: | |
| 293 version_info = '' | |
| 294 return '%s://%s/%s%s' % (uri.scheme, obj.bucket.name, obj.name, | |
| 295 version_info) | |
| 296 | |
| 297 def _PrintInfoAboutBucketListingRef(self, bucket_listing_ref, listing_style): | |
| 298 """Print listing info for given bucket_listing_ref. | |
| 299 | |
| 300 Args: | |
| 301 bucket_listing_ref: BucketListing being listed. | |
| 302 listing_style: ListingStyle enum describing type of output desired. | |
| 303 | |
| 304 Returns: | |
| 305 Tuple (number of objects, | |
| 306 object length, if listing_style is one of the long listing formats) | |
| 307 | |
| 308 Raises: | |
| 309 Exception: if calling bug encountered. | |
| 310 """ | |
| 311 uri = bucket_listing_ref.GetUri() | |
| 312 obj = bucket_listing_ref.GetKey() | |
| 313 uri_str = self._UriStrForObj(uri, obj) | |
| 314 if listing_style == ListingStyle.SHORT: | |
| 315 print uri_str.encode('utf-8') | |
| 316 return (1, 0) | |
| 317 elif listing_style == ListingStyle.LONG: | |
| 318 # Exclude timestamp fractional secs (example: 2010-08-23T12:46:54.187Z). | |
| 319 timestamp = obj.last_modified[:19].decode('utf8').encode('ascii') | |
| 320 if not isinstance(obj, DeleteMarker): | |
| 321 if self.all_versions: | |
| 322 print '%10s %s %s meta_generation=%s' % ( | |
| 323 obj.size, timestamp, uri_str.encode('utf-8'), obj.meta_generation) | |
| 324 else: | |
| 325 print '%10s %s %s' % (obj.size, timestamp, uri_str.encode('utf-8')) | |
| 326 return (1, obj.size) | |
| 327 else: | |
| 328 if self.all_versions: | |
| 329 print '%10s %s %s meta_generation=%s' % ( | |
| 330 0, timestamp, uri_str.encode('utf-8'), obj.meta_generation) | |
| 331 else: | |
| 332 print '%10s %s %s' % (0, timestamp, uri_str.encode('utf-8')) | |
| 333 return (0, 1) | |
| 334 elif listing_style == ListingStyle.LONG_LONG: | |
| 335 # Run in a try/except clause so we can continue listings past | |
| 336 # access-denied errors (which can happen because user may have READ | |
| 337 # permission on object and thus see the bucket listing data, but lack | |
| 338 # FULL_CONTROL over individual objects and thus not be able to read | |
| 339 # their ACLs). | |
| 340 try: | |
| 341 print '%s:' % uri_str.encode('utf-8') | |
| 342 suri = self.suri_builder.StorageUri(uri_str) | |
| 343 obj = suri.get_key(False) | |
| 344 print '\tCreation time:\t%s' % obj.last_modified | |
| 345 if obj.cache_control: | |
| 346 print '\tCache-Control:\t%s' % obj.cache_control | |
| 347 if obj.content_disposition: | |
| 348 print '\tContent-Disposition:\t%s' % obj.content_disposition | |
| 349 if obj.content_encoding: | |
| 350 print '\tContent-Encoding:\t%s' % obj.content_encoding | |
| 351 if obj.content_language: | |
| 352 print '\tContent-Language:\t%s' % obj.content_language | |
| 353 print '\tContent-Length:\t%s' % obj.size | |
| 354 print '\tContent-Type:\t%s' % obj.content_type | |
| 355 if obj.metadata: | |
| 356 prefix = uri.get_provider().metadata_prefix | |
| 357 for name in obj.metadata: | |
| 358 print '\t%s%s:\t%s' % (prefix, name, obj.metadata[name]) | |
| 359 print '\tETag:\t\t%s' % obj.etag.strip('"\'') | |
| 360 print '\tACL:\t\t%s' % (suri.get_acl(False, self.headers)) | |
| 361 return (1, obj.size) | |
| 362 except boto.exception.GSResponseError as e: | |
| 363 if e.status == 403: | |
| 364 print ('\tACL:\t\tACCESS DENIED. Note: you need FULL_CONTROL ' | |
| 365 'permission\n\t\t\ton the object to read its ACL.') | |
| 366 return (1, obj.size) | |
| 367 else: | |
| 368 raise e | |
| 369 else: | |
| 370 raise Exception('Unexpected ListingStyle(%s)' % listing_style) | |
| 371 | |
| 372 def _ExpandUriAndPrintInfo(self, uri, listing_style, should_recurse=False): | |
| 373 """ | |
| 374 Expands wildcards and directories/buckets for uri as needed, and | |
| 375 calls _PrintInfoAboutBucketListingRef() on each. | |
| 376 | |
| 377 Args: | |
| 378 uri: StorageUri being listed. | |
| 379 listing_style: ListingStyle enum describing type of output desired. | |
| 380 should_recurse: bool indicator of whether to expand recursively. | |
| 381 | |
| 382 Returns: | |
| 383 Tuple (number of matching objects, number of bytes across these objects). | |
| 384 """ | |
| 385 # We do a two-level loop, with the outer loop iterating level-by-level from | |
| 386 # blrs_to_expand, and the inner loop iterating the matches at the current | |
| 387 # level, printing them, and adding any new subdirs that need expanding to | |
| 388 # blrs_to_expand (to be picked up in the next outer loop iteration). | |
| 389 blrs_to_expand = [BucketListingRef(uri)] | |
| 390 num_objs = 0 | |
| 391 num_bytes = 0 | |
| 392 expanding_top_level = True | |
| 393 printed_one = False | |
| 394 num_expanded_blrs = 0 | |
| 395 while len(blrs_to_expand): | |
| 396 if printed_one: | |
| 397 print | |
| 398 blr = blrs_to_expand.pop(0) | |
| 399 if blr.HasKey(): | |
| 400 blr_iterator = iter([blr]) | |
| 401 elif blr.HasPrefix(): | |
| 402 # Bucket subdir from a previous iteration. Print "header" line only if | |
| 403 # we're listing more than one subdir (or if it's a recursive listing), | |
| 404 # to be consistent with the way UNIX ls works. | |
| 405 if num_expanded_blrs > 1 or should_recurse: | |
| 406 print '%s:' % blr.GetUriString().encode('utf-8') | |
| 407 printed_one = True | |
| 408 blr_iterator = self.WildcardIterator('%s/*' % | |
| 409 blr.GetRStrippedUriString(), | |
| 410 all_versions=self.all_versions) | |
| 411 elif blr.NamesBucket(): | |
| 412 blr_iterator = self.WildcardIterator('%s*' % blr.GetUriString(), | |
| 413 all_versions=self.all_versions) | |
| 414 else: | |
| 415 # This BLR didn't come from a bucket listing. This case happens for | |
| 416 # BLR's instantiated from a user-provided URI. | |
| 417 blr_iterator = PluralityCheckableIterator( | |
| 418 _UriOnlyBlrExpansionIterator( | |
| 419 self, blr, all_versions=self.all_versions)) | |
| 420 if blr_iterator.is_empty() and not ContainsWildcard(uri): | |
| 421 raise CommandException('No such object %s' % uri) | |
| 422 for cur_blr in blr_iterator: | |
| 423 num_expanded_blrs = num_expanded_blrs + 1 | |
| 424 if cur_blr.HasKey(): | |
| 425 # Object listing. | |
| 426 (no, nb) = self._PrintInfoAboutBucketListingRef( | |
| 427 cur_blr, listing_style) | |
| 428 num_objs += no | |
| 429 num_bytes += nb | |
| 430 printed_one = True | |
| 431 else: | |
| 432 # Subdir listing. If we're at the top level of a bucket subdir | |
| 433 # listing don't print the list here (corresponding to how UNIX ls | |
| 434 # dir just prints its contents, not the name followed by its | |
| 435 # contents). | |
| 436 if (expanding_top_level and not uri.names_bucket()) or should_recurse: | |
| 437 if cur_blr.GetUriString().endswith('//'): | |
| 438 # Expand gs://bucket// into gs://bucket//* so we don't infinite | |
| 439 # loop. This case happens when user has uploaded an object whose | |
| 440 # name begins with a /. | |
| 441 cur_blr = BucketListingRef(self.suri_builder.StorageUri( | |
| 442 '%s*' % cur_blr.GetUriString()), None, None, cur_blr.headers) | |
| 443 blrs_to_expand.append(cur_blr) | |
| 444 # Don't include the subdir name in the output if we're doing a | |
| 445 # recursive listing, as it will be printed as 'subdir:' when we get | |
| 446 # to the prefix expansion, the next iteration of the main loop. | |
| 447 else: | |
| 448 if listing_style == ListingStyle.LONG: | |
| 449 print '%-33s%s' % ( | |
| 450 '', cur_blr.GetUriString().encode('utf-8')) | |
| 451 else: | |
| 452 print cur_blr.GetUriString().encode('utf-8') | |
| 453 expanding_top_level = False | |
| 454 return (num_objs, num_bytes) | |
| 455 | |
| 456 # Command entry point. | |
| 457 def RunCommand(self): | |
| 458 got_nomatch_errors = False | |
| 459 listing_style = ListingStyle.SHORT | |
| 460 get_bucket_info = False | |
| 461 self.recursion_requested = False | |
| 462 self.all_versions = False | |
| 463 if self.sub_opts: | |
| 464 for o, a in self.sub_opts: | |
| 465 if o == '-a': | |
| 466 self.all_versions = True | |
| 467 elif o == '-b': | |
| 468 get_bucket_info = True | |
| 469 elif o == '-l': | |
| 470 listing_style = ListingStyle.LONG | |
| 471 elif o == '-L': | |
| 472 listing_style = ListingStyle.LONG_LONG | |
| 473 elif o == '-p': | |
| 474 self.proj_id_handler.SetProjectId(a) | |
| 475 elif o == '-r' or o == '-R': | |
| 476 self.recursion_requested = True | |
| 477 | |
| 478 if not self.args: | |
| 479 # default to listing all gs buckets | |
| 480 self.args = ['gs://'] | |
| 481 | |
| 482 total_objs = 0 | |
| 483 total_bytes = 0 | |
| 484 for uri_str in self.args: | |
| 485 uri = self.suri_builder.StorageUri(uri_str) | |
| 486 self.proj_id_handler.FillInProjectHeaderIfNeeded('ls', uri, self.headers) | |
| 487 | |
| 488 if uri.names_provider(): | |
| 489 # Provider URI: use bucket wildcard to list buckets. | |
| 490 for uri in self.WildcardIterator('%s://*' % uri.scheme).IterUris(): | |
| 491 (bucket_objs, bucket_bytes) = self._PrintBucketInfo(uri, | |
| 492 listing_style) | |
| 493 total_bytes += bucket_bytes | |
| 494 total_objs += bucket_objs | |
| 495 elif uri.names_bucket(): | |
| 496 # Bucket URI -> list the object(s) in that bucket. | |
| 497 if get_bucket_info: | |
| 498 # ls -b bucket listing request: List info about bucket(s). | |
| 499 for uri in self.WildcardIterator(uri).IterUris(): | |
| 500 (bucket_objs, bucket_bytes) = self._PrintBucketInfo(uri, | |
| 501 listing_style) | |
| 502 total_bytes += bucket_bytes | |
| 503 total_objs += bucket_objs | |
| 504 else: | |
| 505 # Not -b request: List objects in the bucket(s). | |
| 506 (no, nb) = self._ExpandUriAndPrintInfo(uri, listing_style, | |
| 507 should_recurse=self.recursion_requested) | |
| 508 if no == 0 and ContainsWildcard(uri): | |
| 509 got_nomatch_errors = True | |
| 510 total_objs += no | |
| 511 total_bytes += nb | |
| 512 else: | |
| 513 # URI names an object or object subdir -> list matching object(s) / | |
| 514 # subdirs. | |
| 515 (exp_objs, exp_bytes) = self._ExpandUriAndPrintInfo(uri, listing_style, | |
| 516 should_recurse=self.recursion_requested) | |
| 517 if exp_objs == 0 and ContainsWildcard(uri): | |
| 518 got_nomatch_errors = True | |
| 519 total_bytes += exp_bytes | |
| 520 total_objs += exp_objs | |
| 521 | |
| 522 if total_objs and listing_style != ListingStyle.SHORT: | |
| 523 print ('TOTAL: %d objects, %d bytes (%s)' % | |
| 524 (total_objs, total_bytes, MakeHumanReadable(float(total_bytes)))) | |
| 525 if got_nomatch_errors: | |
| 526 raise CommandException('One or more URIs matched no objects.') | |
| 527 | |
| 528 return 0 | |
| 529 | |
| 530 | |
| 531 class _UriOnlyBlrExpansionIterator: | |
| 532 """ | |
| 533 Iterator that expands a BucketListingRef that contains only a URI (i.e., | |
| 534 didn't come from a bucket listing), yielding BucketListingRefs to which it | |
| 535 expands. This case happens for BLR's instantiated from a user-provided URI. | |
| 536 | |
| 537 Note that we can't use NameExpansionIterator here because it produces an | |
| 538 iteration over the full object names (e.g., expanding "gs://bucket" to | |
| 539 "gs://bucket/dir/o1" and "gs://bucket/dir/o2"), while for the ls command | |
| 540 we need also to see the intermediate directories (like "gs://bucket/dir"). | |
| 541 """ | |
| 542 def __init__(self, command_instance, blr, all_versions=False): | |
| 543 self.command_instance = command_instance | |
| 544 self.blr = blr | |
| 545 self.all_versions=all_versions | |
| 546 | |
| 547 def __iter__(self): | |
| 548 """ | |
| 549 Args: | |
| 550 command_instance: calling instance of Command class. | |
| 551 blr: BucketListingRef to expand. | |
| 552 | |
| 553 Yields: | |
| 554 List of BucketListingRef to which it expands. | |
| 555 """ | |
| 556 # Do a delimited wildcard expansion so we get any matches along with | |
| 557 # whether they are keys or prefixes. That way if bucket contains a key | |
| 558 # 'abcd' and another key 'abce/x.txt' the expansion will return two BLRs, | |
| 559 # the first with HasKey()=True and the second with HasPrefix()=True. | |
| 560 rstripped_versionless_uri_str = self.blr.GetRStrippedUriString() | |
| 561 if ContainsWildcard(rstripped_versionless_uri_str): | |
| 562 for blr in self.command_instance.WildcardIterator( | |
| 563 rstripped_versionless_uri_str, all_versions=self.all_versions): | |
| 564 yield blr | |
| 565 return | |
| 566 # Build a wildcard to expand so CloudWildcardIterator will not just treat it | |
| 567 # as a key and yield the result without doing a bucket listing. | |
| 568 for blr in self.command_instance.WildcardIterator( | |
| 569 rstripped_versionless_uri_str + '*', all_versions=self.all_versions): | |
| 570 # Find the originally specified BucketListingRef in the expanded list (if | |
| 571 # present). Don't just use the expanded list, because it would also | |
| 572 # include objects whose name prefix matches the blr name (because of the | |
| 573 # wildcard match we did above). Note that there can be multiple matches, | |
| 574 # for the case where there's both an object and a subdirectory with the | |
| 575 # same name. | |
| 576 if (blr.GetRStrippedUriString() | |
| 577 == rstripped_versionless_uri_str): | |
| 578 yield blr | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2280023003:
depot_tools: Remove third_party/gsutil (Closed)
