Roll version of gsutil to 4.15.

third_party/gsutil/gslib/commands/cp.py

 # -*- coding: utf-8 -*-
 # Copyright 2011 Google Inc. All Rights Reserved.
 # Copyright 2011, Nexenta Systems Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Implementation of Unix-like cp command for cloud storage providers."""
 
 from __future__ import absolute_import
 
 import os
 import time
 import traceback
 
 from gslib import copy_helper
 from gslib.cat_helper import CatHelper
-from gslib.cloud_api import AccessDeniedException
-from gslib.cloud_api import NotFoundException
 from gslib.command import Command
 from gslib.command_argument import CommandArgument
 from gslib.commands.compose import MAX_COMPONENT_COUNT
 from gslib.copy_helper import CreateCopyHelperOpts
 from gslib.copy_helper import ItemExistsError
 from gslib.copy_helper import Manifest
 from gslib.copy_helper import PARALLEL_UPLOAD_TEMP_NAMESPACE
 from gslib.copy_helper import SkipUnsupportedObjectError
 from gslib.cs_api_map import ApiSelector
 from gslib.exception import CommandException
@@ skipping 164 matching lines @@
   in the cloud, which may take some time. Such operations can be resumed with
   the same command if they are interrupted, so long as the command parameters
   are identical. 
 
   Note that by default, the gsutil cp command does not copy the object
   ACL to the new object, and instead will use the default bucket ACL (see
   "gsutil help defacl").  You can override this behavior with the -p
   option (see OPTIONS below).
 
   One additional note about copying in the cloud: If the destination bucket has
-  versioning enabled, gsutil cp will copy all versions of the source object(s).
-  For example:
+  versioning enabled, gsutil cp will by default copy only live versions of the
+  source object(s). For example:
 
     gsutil cp gs://bucket1/obj gs://bucket2
 
-  will cause all versions of gs://bucket1/obj to be copied to gs://bucket2.
+  will cause only the single live version of of gs://bucket1/obj to be copied
+  to gs://bucket2, even if there are archived versions of gs://bucket1/obj. To
+  also copy archived versions, use the -A flag:
+
+    gsutil cp -A gs://bucket1/obj gs://bucket2
+
+  The gsutil -m flag is disallowed when using the cp -A flag, to ensure that
+  version ordering is preserved.
 """
 
 _CHECKSUM_VALIDATION_TEXT = """
 <B>CHECKSUM VALIDATION</B>
   At the end of every upload or download the gsutil cp command validates that
   the checksum it computes for the source file/object matches the checksum
   the service computes. If the checksums do not match, gsutil will delete the
   corrupted object and print a warning message. This very rarely happens, but
   if it does, please contact gs-team@google.com.
 
@@ skipping 65 matching lines @@
   MiB. You do not need to specify any special command line options to make this
   happen. If your upload is interrupted you can restart the upload by running
   the same cp command that you ran to start the upload. Until the upload
   has completed successfully, it will not be visible at the destination object
   and will not replace any existing object the upload is intended to overwrite.
   (However, see the section on PARALLEL COMPOSITE UPLOADS, which may leave
   temporary component objects in place during the upload process.)
 
   Similarly, gsutil automatically performs resumable downloads (using HTTP
   standard Range GET operations) whenever you use the cp command, unless the
-  destination is a stream or null. In this case the partially downloaded file
-  will be visible as soon as it starts being written. Thus, before you attempt
-  to use any files downloaded by gsutil you should make sure the download
-  completed successfully, by checking the exit status from the gsutil command.
-  This can be done in a bash script, for example, by doing:
+  destination is a stream or null. In this case, a partially downloaded
+  temporary file will be visible in the destination directory. Upon completion,
+  the original file is deleted and overwritten with the downloaded contents.
 
-     gsutil cp gs://your-bucket/your-object ./local-file
-     if [ "$status" -ne "0" ] ; then
-       << Code that handles failures >>
-     fi
-
-  Resumable uploads and downloads store some state information in a file
+  Resumable uploads and downloads store some state information in a files
   in ~/.gsutil named by the destination object or file. If you attempt to
   resume a transfer from a machine with a different directory, the transfer
   will start over from scratch.
 
   See also "gsutil help prod" for details on using resumable transfers
   in production.
 """
 
 _STREAMING_TRANSFERS_TEXT = """
 <B>STREAMING TRANSFERS</B>
@@ skipping 10 matching lines @@
   (say, more than 100 MiB) it is recommended to write the data to a local file
   and then copy that file to the cloud rather than streaming it (and similarly
   for large downloads).
 
   WARNING: When performing streaming transfers gsutil does not compute a
   checksum of the uploaded or downloaded data.  Therefore, we recommend that
   users either perform their own validation of the data or use non-streaming
   transfers (which perform integrity checking automatically).
 """
 
+_SLICED_OBJECT_DOWNLOADS_TEXT = """
+<B>SLICED OBJECT DOWNLOADS</B>
+  gsutil automatically uses HTTP Range GET requests to perform "sliced"
+  downloads in parallel for downloads of large objects. This means that, if
+  enabled, disk space for the temporary download destination file will be
+  pre-allocated and byte ranges (slices) within the file will be downloaded in
+  parallel. Once all slices have completed downloading, the temporary file will
+  be renamed to the destination file. No additional local disk space is
+  required for this operation.
+
+  This feature is only available for Google Cloud Storage objects because it
+  requires a fast composable checksum that can be used to verify the data
+  integrity of the slices. Thus, using sliced object downloads also requires a
+  compiled crcmod (see "gsutil help crcmod") on the machine performing the
+  download. If compiled crcmod is not available, normal download will instead
+  be used.
+
+  Note: since sliced object downloads cause multiple writes to occur at various
+  locations on disk, this can degrade performance for disks with slow seek
+  times, especially for large numbers of slices. While the default number of
+  slices is small to avoid this, sliced object download can be completely
+  disabled by setting the "sliced_object_download_threshold" variable in the
+  .boto config file to 0.
+"""
+
 _PARALLEL_COMPOSITE_UPLOADS_TEXT = """
 <B>PARALLEL COMPOSITE UPLOADS</B>
   gsutil can automatically use
   `object composition <https://developers.google.com/storage/docs/composite-objects>`_
   to perform uploads in parallel for large, local files being uploaded to Google
   Cloud Storage. This means that, if enabled (see next paragraph), a large file
   will be split into component pieces that will be uploaded in parallel. Those
   components will then be composed in the cloud, and the temporary components in
   the cloud will be deleted after successful composition. No additional local
   disk space is required for this operation.
 
   Using parallel composite uploads presents a tradeoff between upload
   performance and download configuration: If you enable parallel composite
   uploads your uploads will run faster, but someone will need to install a
   compiled crcmod (see "gsutil help crcmod") on every machine where objects are
   downloaded by gsutil or other Python applications. For some distributions this
   is easy (e.g., it comes pre-installed on MacOS), but in some cases users have
   found it difficult. Because of this at present parallel composite uploads are
   disabled by default. Google is actively working with a number of the Linux
   distributions to get crcmod included with the stock distribution. Once that is
   done we will re-enable parallel composite uploads by default in gsutil.
 
+  Parallel composite uploads should not be used with NEARLINE storage
+  class buckets, as doing this would incur an early deletion charge for each
+  component object.
+
   To try parallel composite uploads you can run the command:
 
     gsutil -o GSUtil:parallel_composite_upload_threshold=150M cp bigfile gs://your-bucket
 
   where bigfile is larger than 150 MiB. When you do this notice that the upload
   progress indicator continuously updates for several different uploads at once
   (corresponding to each of the sections of the file being uploaded in
   parallel), until the parallel upload completes. If you then want to enable
   parallel composite uploads for all of your future uploads (notwithstanding the
   caveats mentioned earlier), you can uncomment and set the
@@ skipping 80 matching lines @@
     export TMPDIR=/some/directory
 
   On Windows 7 you can change the TMPDIR environment variable from Start ->
   Computer -> System -> Advanced System Settings -> Environment Variables.
   You need to reboot after making this change for it to take effect. (Rebooting
   is not necessary after running the export command on Linux and MacOS.)
 """
 
 _OPTIONS_TEXT = """
 <B>OPTIONS</B>
-  -a canned_acl   Sets named canned_acl when uploaded objects created. See
-                  'gsutil help acls' for further details.
+  -a canned_acl  Sets named canned_acl when uploaded objects created. See
+                 'gsutil help acls' for further details.
+
+  -A             Copy all source versions from a source buckets/folders.
+                 If not set, only the live version of each source object is
+                 copied. Note: this option is only useful when the destination
+                 bucket has versioning enabled.
 
   -c             If an error occurs, continue to attempt to copy the remaining
                  files. If any copies were unsuccessful, gsutil's exit status
                  will be non-zero even if this flag is set. This option is
                  implicitly set when running "gsutil -m cp...". Note: -c only
                  applies to the actual copying operation. If an error occurs
                  while iterating over the files in the local directory (e.g.,
                  invalid Unicode file name) gsutil will print an error message
                  and abort.
 
@@ skipping 87 matching lines @@
                  together.
 
   -R, -r         Causes directories, buckets, and bucket subdirectories to be
                  copied recursively. If you neglect to use this option for
                  an upload, gsutil will copy any files it finds and skip any
                  directories. Similarly, neglecting to specify -r for a download
                  will cause gsutil to copy any objects at the current bucket
                  directory level, and skip any subdirectories.
 
   -U             Skip objects with unsupported object types instead of failing.
-                 Unsupported object types are s3 glacier objects.
+                 Unsupported object types are Amazon S3 Objects in the GLACIER
+                 storage class.
 
   -v             Requests that the version-specific URL for each uploaded object
                  be printed. Given this URL you can make future upload requests
                  that are safe in the face of concurrent updates, because Google
                  Cloud Storage will refuse to perform the update if the current
                  object version doesn't match the version-specific URL. See
                  'gsutil help versions' for more details.
 
   -z <ext,...>   Applies gzip content-encoding to file uploads with the given
                  extensions. This is useful when uploading files with
@@ skipping 32 matching lines @@
 
 _DETAILED_HELP_TEXT = '\n\n'.join([_SYNOPSIS_TEXT,
                                    _DESCRIPTION_TEXT,
                                    _NAME_CONSTRUCTION_TEXT,
                                    _SUBDIRECTORIES_TEXT,
                                    _COPY_IN_CLOUD_TEXT,
                                    _CHECKSUM_VALIDATION_TEXT,
                                    _RETRY_HANDLING_TEXT,
                                    _RESUMABLE_TRANSFERS_TEXT,
                                    _STREAMING_TRANSFERS_TEXT,
+                                   _SLICED_OBJECT_DOWNLOADS_TEXT,
                                    _PARALLEL_COMPOSITE_UPLOADS_TEXT,
                                    _CHANGING_TEMP_DIRECTORIES_TEXT,
                                    _OPTIONS_TEXT])
 
 
-CP_SUB_ARGS = 'a:cDeIL:MNnprRtUvz:'
+CP_SUB_ARGS = 'a:AcDeIL:MNnprRtUvz:'
 
 
 def _CopyFuncWrapper(cls, args, thread_state=None):
   cls.CopyFunc(args, thread_state=thread_state)
 
 
 def _CopyExceptionHandler(cls, e):
   """Simple exception handler to allow post-completion status."""
   cls.logger.error(str(e))
   cls.op_failure_count += 1
@@ skipping 80 matching lines @@
 
     # Various GUI tools (like the GCS web console) create placeholder objects
     # ending with '/' when the user creates an empty directory. Normally these
     # tools should delete those placeholders once objects have been written
     # "under" the directory, but sometimes the placeholders are left around. We
     # need to filter them out here, otherwise if the user tries to rsync from
     # GCS to a local directory it will result in a directory/file conflict
     # (e.g., trying to download an object called "mydata/" where the local
     # directory "mydata" exists).
     if IsCloudSubdirPlaceholder(exp_src_url):
-      self.logger.info('Skipping cloud sub-directory placeholder object (%s) '
-                       'because such objects aren\'t needed in (and would '
-                       'interfere with) directories in the local file system',
-                       exp_src_url)
+      # We used to output the message 'Skipping cloud sub-directory placeholder
+      # object...' but we no longer do so because it caused customer confusion.
       return
 
     if copy_helper_opts.use_manifest and self.manifest.WasSuccessful(
         exp_src_url.url_string):
       return
 
     if copy_helper_opts.perform_mv:
       if name_expansion_result.names_container:
         # Use recursion_requested when performing name expansion for the
         # directory mv case so we can determine if any of the source URLs are
@@ skipping 33 matching lines @@
     elapsed_time = bytes_transferred = 0
     try:
       if copy_helper_opts.use_manifest:
         self.manifest.Initialize(
             exp_src_url.url_string, dst_url.url_string)
       (elapsed_time, bytes_transferred, result_url, md5) = (
           copy_helper.PerformCopy(
               self.logger, exp_src_url, dst_url, gsutil_api,
               self, _CopyExceptionHandler, allow_splitting=True,
               headers=self.headers, manifest=self.manifest,
-              gzip_exts=self.gzip_exts, test_method=self.test_method))
+              gzip_exts=self.gzip_exts))
       if copy_helper_opts.use_manifest:
         if md5:
           self.manifest.Set(exp_src_url.url_string, 'md5', md5)
         self.manifest.SetResult(
             exp_src_url.url_string, bytes_transferred, 'OK')
       if copy_helper_opts.print_ver:
         # Some cases don't return a version-specific URL (e.g., if destination
         # is a file).
         self.logger.info('Created: %s', result_url)
     except ItemExistsError:
@@ skipping 62 matching lines @@
       url_strs = StdinIterator()
     else:
       if len(self.args) < 2:
         raise CommandException('Wrong number of arguments for "cp" command.')
       url_strs = self.args[:-1]
 
     (self.exp_dst_url, self.have_existing_dst_container) = (
         copy_helper.ExpandUrlToSingleBlr(self.args[-1], self.gsutil_api,
                                          self.debug, self.project_id))
 
-    # If the destination bucket has versioning enabled iterate with
-    # all_versions=True. That way we'll copy all versions if the source bucket
-    # is versioned; and by leaving all_versions=False if the destination bucket
-    # has versioning disabled we will avoid copying old versions all to the same
-    # un-versioned destination object.
-    all_versions = False
-    try:
-      bucket = self._GetBucketWithVersioningConfig(self.exp_dst_url)
-      if bucket and bucket.versioning and bucket.versioning.enabled:
-        all_versions = True
-    except AccessDeniedException:
-      # This happens (in the XML API only) if the user doesn't have OWNER access
-      # on the bucket (needed to check if versioning is enabled). In this case
-      # fall back to copying all versions (which can be inefficient for the
-      # reason noted in the comment above). We don't try to warn the user
-      # because that would result in false positive warnings (since we can't
-      # check if versioning is enabled on the destination bucket).
-      #
-      # For JSON, we will silently not return versioning if we don't have
-      # access.
-      all_versions = True
-
     name_expansion_iterator = NameExpansionIterator(
         self.command_name, self.debug,
         self.logger, self.gsutil_api, url_strs,
         self.recursion_requested or copy_helper_opts.perform_mv,
-        project_id=self.project_id, all_versions=all_versions,
+        project_id=self.project_id, all_versions=self.all_versions,
         continue_on_error=self.continue_on_error or self.parallel_operations)
 
     # Use a lock to ensure accurate statistics in the face of
     # multi-threading/multi-processing.
     self.stats_lock = CreateLock()
 
     # Tracks if any copies failed.
     self.op_failure_count = 0
 
     # Start the clock.
@@ skipping 29 matching lines @@
     if self.debug == 3:
       # Note that this only counts the actual GET and PUT bytes for the copy
       # - not any transfers for doing wildcard expansion, the initial
       # HEAD/GET request performed to get the object metadata, etc.
       if self.total_bytes_transferred != 0:
         self.logger.info(
             'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)',
             self.total_bytes_transferred, self.total_elapsed_time,
             MakeHumanReadable(self.total_bytes_per_second))
     if self.op_failure_count:
-      plural_str = 's' if self.op_failure_count else ''
+      plural_str = 's' if self.op_failure_count > 1 else ''
       raise CommandException('%d file%s/object%s could not be transferred.' % (
           self.op_failure_count, plural_str, plural_str))
 
     return 0
 
   def _ParseOpts(self):
     perform_mv = False
     # exclude_symlinks is handled by Command parent class, so save in Command
     # state rather than CopyHelperOpts.
     self.exclude_symlinks = False
     no_clobber = False
     # continue_on_error is handled by Command parent class, so save in Command
     # state rather than CopyHelperOpts.
     self.continue_on_error = False
     daisy_chain = False
     read_args_from_stdin = False
     print_ver = False
     use_manifest = False
     preserve_acl = False
     canned_acl = None
     # canned_acl is handled by a helper function in parent
     # Command class, so save in Command state rather than CopyHelperOpts.
     self.canned = None
 
+    self.all_versions = False
+
     self.skip_unsupported_objects = False
 
     # Files matching these extensions should be gzipped before uploading.
     self.gzip_exts = []
 
     test_callback_file = None
 
     # self.recursion_requested initialized in command.py (so can be checked
     # in parent class for all commands).
     self.manifest = None
     if self.sub_opts:
       for o, a in self.sub_opts:
         if o == '-a':
           canned_acl = a
           self.canned = True
+        if o == '-A':
+          self.all_versions = True
         if o == '-c':
           self.continue_on_error = True
         elif o == '-D':
           daisy_chain = True
         elif o == '-e':
           self.exclude_symlinks = True
         elif o == '--testcallbackfile':
           # File path of a pickled class that implements ProgressCallback.call.
           # Used for testing transfer interruptions and resumes.
           test_callback_file = a
@@ skipping 16 matching lines @@
           self.recursion_requested = True
         elif o == '-U':
           self.skip_unsupported_objects = True
         elif o == '-v':
           print_ver = True
         elif o == '-z':
           self.gzip_exts = [x.strip() for x in a.split(',')]
     if preserve_acl and canned_acl:
       raise CommandException(
           'Specifying both the -p and -a options together is invalid.')
+    if self.all_versions and self.parallel_operations:
+      raise CommandException(
+          'The gsutil -m option is not supported with the cp -A flag, to '
+          'ensure that object version ordering is preserved. Please re-run '
+          'the command without the -m option.')
     return CreateCopyHelperOpts(
         perform_mv=perform_mv,
         no_clobber=no_clobber,
         daisy_chain=daisy_chain,
         read_args_from_stdin=read_args_from_stdin,
         print_ver=print_ver,
         use_manifest=use_manifest,
         preserve_acl=preserve_acl,
         canned_acl=canned_acl,
         skip_unsupported_objects=self.skip_unsupported_objects,
         test_callback_file=test_callback_file)
-
-  def _GetBucketWithVersioningConfig(self, exp_dst_url):
-    """Gets versioning config for a bucket and ensures that it exists.
-
-    Args:
-      exp_dst_url: Wildcard-expanded destination StorageUrl.
-
-    Raises:
-      AccessDeniedException: if there was a permissions problem accessing the
-                             bucket or its versioning config.
-      CommandException: if URL refers to a cloud bucket that does not exist.
-
-    Returns:
-      apitools Bucket with versioning configuration.
-    """
-    bucket = None
-    if exp_dst_url.IsCloudUrl() and exp_dst_url.IsBucket():
-      try:
-        bucket = self.gsutil_api.GetBucket(
-            exp_dst_url.bucket_name, provider=exp_dst_url.scheme,
-            fields=['versioning'])
-      except AccessDeniedException, e:
-        raise
-      except NotFoundException, e:
-        raise CommandException('Destination bucket %s does not exist.' %
-                               exp_dst_url)
-      except Exception, e:
-        raise CommandException('Error retrieving destination bucket %s: %s' %
-                               (exp_dst_url, e.message))
-      return bucket