Roll version of gsutil to 4.15.

third_party/gsutil/gslib/commands/rsync.py

 # -*- coding: utf-8 -*-
 # Copyright 2014 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
@@ skipping 13 matching lines @@
 import re
 import tempfile
 import textwrap
 import traceback
 import urllib
 
 from boto import config
 import crcmod
 
 from gslib import copy_helper
+from gslib.bucket_listing_ref import BucketListingObject
 from gslib.cloud_api import NotFoundException
 from gslib.command import Command
 from gslib.command import DummyArgChecker
 from gslib.command_argument import CommandArgument
 from gslib.copy_helper import CreateCopyHelperOpts
 from gslib.copy_helper import SkipUnsupportedObjectError
 from gslib.cs_api_map import ApiSelector
 from gslib.exception import CommandException
 from gslib.hashing_helper import CalculateB64EncodedCrc32cFromContents
 from gslib.hashing_helper import CalculateB64EncodedMd5FromContents
@@ skipping 202 matching lines @@
   you are running a Python library for computing CRC32C, which is much slower
   than using the compiled code. For information on getting a compiled CRC32C
   implementation, see 'gsutil help crc32c'.
 
 
 <B>LIMITATIONS</B>
   1. The gsutil rsync command doesn't make the destination object's timestamps
      match those of the source object (it can't; timestamp setting is not
      allowed by the GCS API).
 
-  2. The gsutil rsync command ignores versioning, synchronizing only the live
-     object versions in versioned buckets.
+  2. The gsutil rsync command considers only the current object generations in
+     the source and destination buckets when deciding what to copy / delete. If
+     versioning is enabled in the destination bucket then gsutil rsync's
+     overwriting or deleting objects will end up creating versions, but the
+     command doesn't try to make the archived generations match in the source
+     and destination buckets.
+
 
 
 <B>OPTIONS</B>
   -c            Causes the rsync command to compute checksums for files if the
                 size of source and destination match, and then compare
                 checksums.  This option increases local disk I/O and run time
                 if either src_url or dst_url are on the local file system.
 
   -C            If an error occurs, continue to attempt to copy the remaining
                 files. If errors occurred, gsutil's exit status will be non-zero
@@ skipping 28 matching lines @@
                 rsync -p if you want all objects in the destination bucket to
                 end up with the same ACL by setting a default object ACL on that
                 bucket instead of using rsync -p. See 'help gsutil defacl'.
 
   -R, -r        Causes directories, buckets, and bucket subdirectories to be
                 synchronized recursively. If you neglect to use this option
                 gsutil will make only the top-level directory in the source
                 and destination URLs match, skipping any sub-directories.
 
   -U            Skip objects with unsupported object types instead of failing.
-                Unsupported object types are s3 glacier objects.
+                Unsupported object types are Amazon S3 Objects in the GLACIER
+                storage class.
 
   -x pattern    Causes files/objects matching pattern to be excluded, i.e., any
                 matching files/objects will not be copied or deleted. Note that
                 the pattern is a Python regular expression, not a wildcard (so,
                 matching any string ending in 'abc' would be specified using
                 '.*abc' rather than '*abc'). Note also that the exclude path is
                 always relative (similar to Unix rsync or tar exclude options).
                 For example, if you run the command:
 
                   gsutil rsync -x 'data./.*\\.txt' dir gs://my-bucket
@@ skipping 148 matching lines @@
     # futile or could result in data loss - for example:
     #     gsutil rsync -d gs://non-existent-bucket ./localdir
     # would delete files from localdir.
     cls.logger.error(
         'Caught non-retryable exception while listing %s: %s' %
         (base_url_str, e))
     cls.non_retryable_listing_failures = 1
   out_file.close()
 
 
+def _LocalDirIterator(base_url):
+  """A generator that yields a BLR for each file in a local directory.
+
+     We use this function instead of WildcardIterator for listing a local
+     directory without recursion, because the glob.globi implementation called
+     by WildcardIterator skips "dot" files (which we don't want to do when
+     synchronizing to or from a local directory).
+
+  Args:
+    base_url: URL for the directory over which to iterate.
+
+  Yields:
+    BucketListingObject for each file in the directory.
+  """
+  for filename in os.listdir(base_url.object_name):
+    filename = os.path.join(base_url.object_name, filename)
+    if os.path.isfile(filename):
+      yield BucketListingObject(StorageUrlFromString(filename), None)
+
+
 def _FieldedListingIterator(cls, gsutil_api, base_url_str, desc):
   """Iterator over base_url_str formatting output per _BuildTmpOutputLine.
 
   Args:
     cls: Command instance.
     gsutil_api: gsutil Cloud API instance to use for bucket listing.
     base_url_str: The top-level URL string over which to iterate.
     desc: 'source' or 'destination'.
 
   Yields:
     Output line formatted per _BuildTmpOutputLine.
   """
-  if cls.recursion_requested:
-    wildcard = '%s/**' % base_url_str.rstrip('/\\')
+  base_url = StorageUrlFromString(base_url_str)
+  if base_url.scheme == 'file' and not cls.recursion_requested:
+    iterator = _LocalDirIterator(base_url)
   else:
-    wildcard = '%s/*' % base_url_str.rstrip('/\\')
+    if cls.recursion_requested:
+      wildcard = '%s/**' % base_url_str.rstrip('/\\')
+    else:
+      wildcard = '%s/*' % base_url_str.rstrip('/\\')
+    iterator = CreateWildcardIterator(
+        wildcard, gsutil_api, debug=cls.debug,
+        project_id=cls.project_id).IterObjects(
+            # Request just the needed fields, to reduce bandwidth usage.
+            bucket_listing_fields=['crc32c', 'md5Hash', 'name', 'size'])
+
   i = 0
-  for blr in CreateWildcardIterator(
-      wildcard, gsutil_api, debug=cls.debug,
-      project_id=cls.project_id).IterObjects(
-          # Request just the needed fields, to reduce bandwidth usage.
-          bucket_listing_fields=['crc32c', 'md5Hash', 'name', 'size']):
+  for blr in iterator:
     # Various GUI tools (like the GCS web console) create placeholder objects
     # ending with '/' when the user creates an empty directory. Normally these
     # tools should delete those placeholders once objects have been written
     # "under" the directory, but sometimes the placeholders are left around.
     # We need to filter them out here, otherwise if the user tries to rsync
     # from GCS to a local directory it will result in a directory/file
     # conflict (e.g., trying to download an object called "mydata/" where the
     # local directory "mydata" exists).
     url = blr.storage_url
     if IsCloudSubdirPlaceholder(url, blr=blr):
-      cls.logger.info('Skipping cloud sub-directory placeholder object (%s) '
-                      'because such objects aren\'t needed in (and would '
-                      'interfere with) directories in the local file system',
-                      url)
+      # We used to output the message 'Skipping cloud sub-directory placeholder
+      # object...' but we no longer do so because it caused customer confusion.
       continue
     if (cls.exclude_symlinks and url.IsFileUrl()
         and os.path.islink(url.object_name)):
       continue
     if cls.exclude_pattern:
       str_to_check = url.url_string[len(base_url_str):]
       if str_to_check.startswith(url.delim):
         str_to_check = str_to_check[1:]
       if cls.exclude_pattern.match(str_to_check):
         continue
@@ skipping 502 matching lines @@
         elif o == '-x':
           if not a:
             raise CommandException('Invalid blank exclude filter')
           try:
             self.exclude_pattern = re.compile(a)
           except re.error:
             raise CommandException('Invalid exclude filter (%s)' % a)
     return CreateCopyHelperOpts(
         preserve_acl=preserve_acl,
         skip_unsupported_objects=self.skip_unsupported_objects)