Revert "Add gsutil 4.13 to telemetry/third_party"

tools/telemetry/third_party/gsutil/gslib/commands/update.py

-# -*- coding: utf-8 -*-
-# Copyright 2011 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Implementation of update command for updating gsutil."""
-
-from __future__ import absolute_import
-
-import os
-import shutil
-import signal
-import stat
-import tarfile
-import tempfile
-import textwrap
-
-import gslib
-from gslib.command import Command
-from gslib.cs_api_map import ApiSelector
-from gslib.exception import CommandException
-from gslib.sig_handling import RegisterSignalHandler
-from gslib.util import CERTIFICATE_VALIDATION_ENABLED
-from gslib.util import CompareVersions
-from gslib.util import GetBotoConfigFileList
-from gslib.util import GSUTIL_PUB_TARBALL
-from gslib.util import IS_CYGWIN
-from gslib.util import IS_WINDOWS
-from gslib.util import LookUpGsutilVersion
-from gslib.util import RELEASE_NOTES_URL
-
-
-_SYNOPSIS = """
-  gsutil update [-f] [-n] [url]
-"""
-
-_DETAILED_HELP_TEXT = ("""
-<B>SYNOPSIS</B>
-""" + _SYNOPSIS + """
-
-
-<B>DESCRIPTION</B>
-  The gsutil update command downloads the latest gsutil release, checks its
-  version, and offers to let you update to it if it differs from the version
-  you're currently running.
-
-  Once you say "Y" to the prompt of whether to install the update, the gsutil
-  update command locates where the running copy of gsutil is installed,
-  unpacks the new version into an adjacent directory, moves the previous version
-  aside, moves the new version to where the previous version was installed,
-  and removes the moved-aside old version. Because of this, users are cautioned
-  not to store data in the gsutil directory, since that data will be lost
-  when you update gsutil. (Some users change directories into the gsutil
-  directory to run the command. We advise against doing that, for this reason.)
-  Note also that the gsutil update command will refuse to run if it finds user
-  data in the gsutil directory.
-
-  By default gsutil update will retrieve the new code from
-  %s, but you can optionally specify a URL to use
-  instead. This is primarily used for distributing pre-release versions of
-  the code to a small group of early test users.
-
-  Note: gsutil periodically checks whether a more recent software update is
-  available. By default this check is performed every 30 days; you can change
-  (or disable) this check by editing the software_update_check_period variable
-  in the .boto config file. Note also that gsutil will only check for software
-  updates if stdin, stdout, and stderr are all connected to a TTY, to avoid
-  interfering with cron jobs, streaming transfers, and other cases where gsutil
-  input or output are redirected from/to files or pipes. Software update
-  periodic checks are also disabled by the gsutil -q option (see
-  'gsutil help options')
-
-
-<B>OPTIONS</B>
-  -f          Forces the update command to offer to let you update, even if you
-              have the most current copy already. This can be useful if you have
-              a corrupted local copy.
-
-  -n          Causes update command to run without prompting [Y/n] whether to
-              continue if an update is available.
-""" % GSUTIL_PUB_TARBALL)
-
-
-class UpdateCommand(Command):
-  """Implementation of gsutil update command."""
-
-  # Command specification. See base class for documentation.
-  command_spec = Command.CreateCommandSpec(
-      'update',
-      command_name_aliases=['refresh'],
-      usage_synopsis=_SYNOPSIS,
-      min_args=0,
-      max_args=1,
-      supported_sub_args='fn',
-      file_url_ok=True,
-      provider_url_ok=False,
-      urls_start_arg=0,
-      gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
-      gs_default_api=ApiSelector.JSON,
-  )
-  # Help specification. See help_provider.py for documentation.
-  help_spec = Command.HelpSpec(
-      help_name='update',
-      help_name_aliases=['refresh'],
-      help_type='command_help',
-      help_one_line_summary='Update to the latest gsutil release',
-      help_text=_DETAILED_HELP_TEXT,
-      subcommand_help_text={},
-  )
-
-  def _DisallowUpdataIfDataInGsutilDir(self):
-    """Disallows the update command if files not in the gsutil distro are found.
-
-    This prevents users from losing data if they are in the habit of running
-    gsutil from the gsutil directory and leaving data in that directory.
-
-    This will also detect someone attempting to run gsutil update from a git
-    repo, since the top-level directory will contain git files and dirs (like
-    .git) that are not distributed with gsutil.
-
-    Raises:
-      CommandException: if files other than those distributed with gsutil found.
-    """
-    # Manifest includes recursive-includes of gslib. Directly add
-    # those to the list here so we will skip them in os.listdir() loop without
-    # having to build deeper handling of the MANIFEST file here. Also include
-    # 'third_party', which isn't present in manifest but gets added to the
-    # gsutil distro by the gsutil submodule configuration; and the MANIFEST.in
-    # and CHANGES.md files.
-    manifest_lines = ['gslib', 'third_party', 'MANIFEST.in', 'CHANGES.md']
-
-    try:
-      with open(os.path.join(gslib.GSUTIL_DIR, 'MANIFEST.in'), 'r') as fp:
-        for line in fp:
-          if line.startswith('include '):
-            manifest_lines.append(line.split()[-1])
-    except IOError:
-      self.logger.warn('MANIFEST.in not found in %s.\nSkipping user data '
-                       'check.\n', gslib.GSUTIL_DIR)
-      return
-
-    # Look just at top-level directory. We don't try to catch data dropped into
-    # subdirs (like gslib) because that would require deeper parsing of
-    # MANFFEST.in, and most users who drop data into gsutil dir do so at the top
-    # level directory.
-    for filename in os.listdir(gslib.GSUTIL_DIR):
-      if filename.endswith('.pyc'):
-        # Ignore compiled code.
-        continue
-      if filename not in manifest_lines:
-        raise CommandException('\n'.join(textwrap.wrap(
-            'A file (%s) that is not distributed with gsutil was found in '
-            'the gsutil directory. The update command cannot run with user '
-            'data in the gsutil directory.' %
-            os.path.join(gslib.GSUTIL_DIR, filename))))
-
-  def _ExplainIfSudoNeeded(self, tf, dirs_to_remove):
-    """Explains what to do if sudo needed to update gsutil software.
-
-    Happens if gsutil was previously installed by a different user (typically if
-    someone originally installed in a shared file system location, using sudo).
-
-    Args:
-      tf: Opened TarFile.
-      dirs_to_remove: List of directories to remove.
-
-    Raises:
-      CommandException: if errors encountered.
-    """
-    # If running under Windows or Cygwin we don't need (or have) sudo.
-    if IS_CYGWIN or IS_WINDOWS:
-      return
-
-    user_id = os.getuid()
-    if os.stat(gslib.GSUTIL_DIR).st_uid == user_id:
-      return
-
-    # Won't fail - this command runs after main startup code that insists on
-    # having a config file.
-    config_file_list = GetBotoConfigFileList()
-    config_files = ' '.join(config_file_list)
-    self._CleanUpUpdateCommand(tf, dirs_to_remove)
-
-    # Pick current protection of each boto config file for command that restores
-    # protection (rather than fixing at 600) to support use cases like how GCE
-    # installs a service account with an /etc/boto.cfg file protected to 644.
-    chmod_cmds = []
-    for config_file in config_file_list:
-      mode = oct(stat.S_IMODE((os.stat(config_file)[stat.ST_MODE])))
-      chmod_cmds.append('\n\tsudo chmod %s %s' % (mode, config_file))
-
-    raise CommandException('\n'.join(textwrap.wrap(
-        'Since it was installed by a different user previously, you will need '
-        'to update using the following commands. You will be prompted for your '
-        'password, and the install will run as "root". If you\'re unsure what '
-        'this means please ask your system administrator for help:')) + (
-            '\n\tsudo chmod 0644 %s\n\tsudo env BOTO_CONFIG="%s" %s update'
-            '%s') % (config_files, config_files, self.gsutil_path,
-                     ' '.join(chmod_cmds)), informational=True)
-
-  # This list is checked during gsutil update by doing a lowercased
-  # slash-left-stripped check. For example "/Dev" would match the "dev" entry.
-  unsafe_update_dirs = [
-      'applications', 'auto', 'bin', 'boot', 'desktop', 'dev',
-      'documents and settings', 'etc', 'export', 'home', 'kernel', 'lib',
-      'lib32', 'library', 'lost+found', 'mach_kernel', 'media', 'mnt', 'net',
-      'null', 'network', 'opt', 'private', 'proc', 'program files', 'python',
-      'root', 'sbin', 'scripts', 'srv', 'sys', 'system', 'tmp', 'users', 'usr',
-      'var', 'volumes', 'win', 'win32', 'windows', 'winnt',
-  ]
-
-  def _EnsureDirsSafeForUpdate(self, dirs):
-    """Raises Exception if any of dirs is known to be unsafe for gsutil update.
-
-    This provides a fail-safe check to ensure we don't try to overwrite
-    or delete any important directories. (That shouldn't happen given the
-    way we construct tmp dirs, etc., but since the gsutil update cleanup
-    uses shutil.rmtree() it's prudent to add extra checks.)
-
-    Args:
-      dirs: List of directories to check.
-
-    Raises:
-      CommandException: If unsafe directory encountered.
-    """
-    for d in dirs:
-      if not d:
-        d = 'null'
-      if d.lstrip(os.sep).lower() in self.unsafe_update_dirs:
-        raise CommandException('EnsureDirsSafeForUpdate: encountered unsafe '
-                               'directory (%s); aborting update' % d)
-
-  def _CleanUpUpdateCommand(self, tf, dirs_to_remove):
-    """Cleans up temp files etc. from running update command.
-
-    Args:
-      tf: Opened TarFile, or None if none currently open.
-      dirs_to_remove: List of directories to remove.
-
-    """
-    if tf:
-      tf.close()
-    self._EnsureDirsSafeForUpdate(dirs_to_remove)
-    for directory in dirs_to_remove:
-      try:
-        shutil.rmtree(directory)
-      except OSError:
-        # Ignore errors while attempting to remove old dirs under Windows. They
-        # happen because of Windows exclusive file locking, and the update
-        # actually succeeds but just leaves the old versions around in the
-        # user's temp dir.
-        if not IS_WINDOWS:
-          raise
-
-  def RunCommand(self):
-    """Command entry point for the update command."""
-
-    if gslib.IS_PACKAGE_INSTALL:
-      raise CommandException(
-          'The update command is only available for gsutil installed from a '
-          'tarball. If you installed gsutil via another method, use the same '
-          'method to update it.')
-
-    if os.environ.get('CLOUDSDK_WRAPPER') == '1':
-      raise CommandException(
-          'The update command is disabled for Cloud SDK installs. Please run '
-          '"gcloud components update" to update it. Note: the Cloud SDK '
-          'incorporates updates to the underlying tools approximately every 2 '
-          'weeks, so if you are attempting to update to a recently created '
-          'release / pre-release of gsutil it may not yet be available via '
-          'the Cloud SDK.')
-
-    https_validate_certificates = CERTIFICATE_VALIDATION_ENABLED
-    if not https_validate_certificates:
-      raise CommandException(
-          'Your boto configuration has https_validate_certificates = False.\n'
-          'The update command cannot be run this way, for security reasons.')
-
-    self._DisallowUpdataIfDataInGsutilDir()
-
-    force_update = False
-    no_prompt = False
-    if self.sub_opts:
-      for o, unused_a in self.sub_opts:
-        if o == '-f':
-          force_update = True
-        if o == '-n':
-          no_prompt = True
-
-    dirs_to_remove = []
-    tmp_dir = tempfile.mkdtemp()
-    dirs_to_remove.append(tmp_dir)
-    os.chdir(tmp_dir)
-
-    if not no_prompt:
-      self.logger.info('Checking for software update...')
-    if self.args:
-      update_from_url_str = self.args[0]
-      if not update_from_url_str.endswith('.tar.gz'):
-        raise CommandException(
-            'The update command only works with tar.gz files.')
-      for i, result in enumerate(self.WildcardIterator(update_from_url_str)):
-        if i > 0:
-          raise CommandException(
-              'Invalid update URL. Must name a single .tar.gz file.')
-        storage_url = result.storage_url
-        if storage_url.IsFileUrl() and not storage_url.IsDirectory():
-          if not force_update:
-            raise CommandException(
-                ('"update" command does not support "file://" URLs without the '
-                 '-f option.'))
-        elif not (storage_url.IsCloudUrl() and storage_url.IsObject()):
-          raise CommandException(
-              'Invalid update object URL. Must name a single .tar.gz file.')
-    else:
-      update_from_url_str = GSUTIL_PUB_TARBALL
-
-    # Try to retrieve version info from tarball metadata; failing that; download
-    # the tarball and extract the VERSION file. The version lookup will fail
-    # when running the update system test, because it retrieves the tarball from
-    # a temp file rather than a cloud URL (files lack the version metadata).
-    tarball_version = LookUpGsutilVersion(self.gsutil_api, update_from_url_str)
-    if tarball_version:
-      tf = None
-    else:
-      tf = self._FetchAndOpenGsutilTarball(update_from_url_str)
-      tf.extractall()
-      with open(os.path.join('gsutil', 'VERSION'), 'r') as ver_file:
-        tarball_version = ver_file.read().strip()
-
-    if not force_update and gslib.VERSION == tarball_version:
-      self._CleanUpUpdateCommand(tf, dirs_to_remove)
-      if self.args:
-        raise CommandException('You already have %s installed.' %
-                               update_from_url_str, informational=True)
-      else:
-        raise CommandException('You already have the latest gsutil release '
-                               'installed.', informational=True)
-
-    if not no_prompt:
-      (_, major) = CompareVersions(tarball_version, gslib.VERSION)
-      if major:
-        print('\n'.join(textwrap.wrap(
-            'This command will update to the "%s" version of gsutil at %s. '
-            'NOTE: This a major new version, so it is strongly recommended '
-            'that you review the release note details at %s before updating to '
-            'this version, especially if you use gsutil in scripts.'
-            % (tarball_version, gslib.GSUTIL_DIR, RELEASE_NOTES_URL))))
-      else:
-        print('This command will update to the "%s" version of\ngsutil at %s'
-              % (tarball_version, gslib.GSUTIL_DIR))
-    self._ExplainIfSudoNeeded(tf, dirs_to_remove)
-
-    if no_prompt:
-      answer = 'y'
-    else:
-      answer = raw_input('Proceed? [y/N] ')
-    if not answer or answer.lower()[0] != 'y':
-      self._CleanUpUpdateCommand(tf, dirs_to_remove)
-      raise CommandException('Not running update.', informational=True)
-
-    if not tf:
-      tf = self._FetchAndOpenGsutilTarball(update_from_url_str)
-
-    # Ignore keyboard interrupts during the update to reduce the chance someone
-    # hitting ^C leaves gsutil in a broken state.
-    RegisterSignalHandler(signal.SIGINT, signal.SIG_IGN)
-
-    # gslib.GSUTIL_DIR lists the path where the code should end up (like
-    # /usr/local/gsutil), which is one level down from the relative path in the
-    # tarball (since the latter creates files in ./gsutil). So, we need to
-    # extract at the parent directory level.
-    gsutil_bin_parent_dir = os.path.normpath(
-        os.path.join(gslib.GSUTIL_DIR, '..'))
-
-    # Extract tarball to a temporary directory in a sibling to GSUTIL_DIR.
-    old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir)
-    new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir)
-    dirs_to_remove.append(old_dir)
-    dirs_to_remove.append(new_dir)
-    self._EnsureDirsSafeForUpdate(dirs_to_remove)
-    try:
-      tf.extractall(path=new_dir)
-    except Exception, e:
-      self._CleanUpUpdateCommand(tf, dirs_to_remove)
-      raise CommandException('Update failed: %s.' % e)
-
-    # For enterprise mode (shared/central) installation, users with
-    # different user/group than the installation user/group must be
-    # able to run gsutil so we need to do some permissions adjustments
-    # here. Since enterprise mode is not not supported for Windows
-    # users, we can skip this step when running on Windows, which
-    # avoids the problem that Windows has no find or xargs command.
-    if not IS_WINDOWS:
-      # Make all files and dirs in updated area owner-RW and world-R, and make
-      # all directories owner-RWX and world-RX.
-      for dirname, subdirs, filenames in os.walk(new_dir):
-        for filename in filenames:
-          fd = os.open(os.path.join(dirname, filename), os.O_RDONLY)
-          os.fchmod(fd, stat.S_IWRITE | stat.S_IRUSR |
-                    stat.S_IRGRP | stat.S_IROTH)
-          os.close(fd)
-        for subdir in subdirs:
-          fd = os.open(os.path.join(dirname, subdir), os.O_RDONLY)
-          os.fchmod(fd, stat.S_IRWXU | stat.S_IXGRP | stat.S_IXOTH |
-                    stat.S_IRGRP | stat.S_IROTH)
-          os.close(fd)
-
-      # Make main gsutil script owner-RWX and world-RX.
-      fd = os.open(os.path.join(new_dir, 'gsutil', 'gsutil'), os.O_RDONLY)
-      os.fchmod(fd, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP |
-                stat.S_IROTH | stat.S_IXOTH)
-      os.close(fd)
-
-    # Move old installation aside and new into place.
-    os.rename(gslib.GSUTIL_DIR, os.path.join(old_dir, 'old'))
-    os.rename(os.path.join(new_dir, 'gsutil'), gslib.GSUTIL_DIR)
-    self._CleanUpUpdateCommand(tf, dirs_to_remove)
-    RegisterSignalHandler(signal.SIGINT, signal.SIG_DFL)
-    self.logger.info('Update complete.')
-    return 0
-
-  def _FetchAndOpenGsutilTarball(self, update_from_url_str):
-    self.command_runner.RunNamedCommand(
-        'cp', [update_from_url_str, 'file://gsutil.tar.gz'], self.headers,
-        self.debug, skip_update_check=True)
-    # Note: tf is closed in _CleanUpUpdateCommand.
-    tf = tarfile.open('gsutil.tar.gz')
-    tf.errorlevel = 1  # So fatal tarball unpack errors raise exceptions.
-    return tf