[catapult] - Copy Telemetry's gsutilz over to third_party.

third_party/gsutil/gslib/commands/update.py

Index: third_party/gsutil/gslib/commands/update.py
diff --git a/third_party/gsutil/gslib/commands/update.py b/third_party/gsutil/gslib/commands/update.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f8cbffcebc76e0ae50e1a9e31caa6d65242b319
--- /dev/null
+++ b/third_party/gsutil/gslib/commands/update.py
@@ -0,0 +1,439 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of update command for updating gsutil."""
+
+from __future__ import absolute_import
+
+import os
+import shutil
+import signal
+import stat
+import tarfile
+import tempfile
+import textwrap
+
+import gslib
+from gslib.command import Command
+from gslib.cs_api_map import ApiSelector
+from gslib.exception import CommandException
+from gslib.sig_handling import RegisterSignalHandler
+from gslib.util import CERTIFICATE_VALIDATION_ENABLED
+from gslib.util import CompareVersions
+from gslib.util import GetBotoConfigFileList
+from gslib.util import GSUTIL_PUB_TARBALL
+from gslib.util import IS_CYGWIN
+from gslib.util import IS_WINDOWS
+from gslib.util import LookUpGsutilVersion
+from gslib.util import RELEASE_NOTES_URL
+
+
+_SYNOPSIS = """
+  gsutil update [-f] [-n] [url]
+"""
+
+_DETAILED_HELP_TEXT = ("""
+<B>SYNOPSIS</B>
+""" + _SYNOPSIS + """
+
+
+<B>DESCRIPTION</B>
+  The gsutil update command downloads the latest gsutil release, checks its
+  version, and offers to let you update to it if it differs from the version
+  you're currently running.
+
+  Once you say "Y" to the prompt of whether to install the update, the gsutil
+  update command locates where the running copy of gsutil is installed,
+  unpacks the new version into an adjacent directory, moves the previous version
+  aside, moves the new version to where the previous version was installed,
+  and removes the moved-aside old version. Because of this, users are cautioned
+  not to store data in the gsutil directory, since that data will be lost
+  when you update gsutil. (Some users change directories into the gsutil
+  directory to run the command. We advise against doing that, for this reason.)
+  Note also that the gsutil update command will refuse to run if it finds user
+  data in the gsutil directory.
+
+  By default gsutil update will retrieve the new code from
+  %s, but you can optionally specify a URL to use
+  instead. This is primarily used for distributing pre-release versions of
+  the code to a small group of early test users.
+
+  Note: gsutil periodically checks whether a more recent software update is
+  available. By default this check is performed every 30 days; you can change
+  (or disable) this check by editing the software_update_check_period variable
+  in the .boto config file. Note also that gsutil will only check for software
+  updates if stdin, stdout, and stderr are all connected to a TTY, to avoid
+  interfering with cron jobs, streaming transfers, and other cases where gsutil
+  input or output are redirected from/to files or pipes. Software update
+  periodic checks are also disabled by the gsutil -q option (see
+  'gsutil help options')
+
+
+<B>OPTIONS</B>
+  -f          Forces the update command to offer to let you update, even if you
+              have the most current copy already. This can be useful if you have
+              a corrupted local copy.
+
+  -n          Causes update command to run without prompting [Y/n] whether to
+              continue if an update is available.
+""" % GSUTIL_PUB_TARBALL)
+
+
+class UpdateCommand(Command):
+  """Implementation of gsutil update command."""
+
+  # Command specification. See base class for documentation.
+  command_spec = Command.CreateCommandSpec(
+      'update',
+      command_name_aliases=['refresh'],
+      usage_synopsis=_SYNOPSIS,
+      min_args=0,
+      max_args=1,
+      supported_sub_args='fn',
+      file_url_ok=True,
+      provider_url_ok=False,
+      urls_start_arg=0,
+      gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
+      gs_default_api=ApiSelector.JSON,
+  )
+  # Help specification. See help_provider.py for documentation.
+  help_spec = Command.HelpSpec(
+      help_name='update',
+      help_name_aliases=['refresh'],
+      help_type='command_help',
+      help_one_line_summary='Update to the latest gsutil release',
+      help_text=_DETAILED_HELP_TEXT,
+      subcommand_help_text={},
+  )
+
+  def _DisallowUpdataIfDataInGsutilDir(self):
+    """Disallows the update command if files not in the gsutil distro are found.
+
+    This prevents users from losing data if they are in the habit of running
+    gsutil from the gsutil directory and leaving data in that directory.
+
+    This will also detect someone attempting to run gsutil update from a git
+    repo, since the top-level directory will contain git files and dirs (like
+    .git) that are not distributed with gsutil.
+
+    Raises:
+      CommandException: if files other than those distributed with gsutil found.
+    """
+    # Manifest includes recursive-includes of gslib. Directly add
+    # those to the list here so we will skip them in os.listdir() loop without
+    # having to build deeper handling of the MANIFEST file here. Also include
+    # 'third_party', which isn't present in manifest but gets added to the
+    # gsutil distro by the gsutil submodule configuration; and the MANIFEST.in
+    # and CHANGES.md files.
+    manifest_lines = ['gslib', 'third_party', 'MANIFEST.in', 'CHANGES.md']
+
+    try:
+      with open(os.path.join(gslib.GSUTIL_DIR, 'MANIFEST.in'), 'r') as fp:
+        for line in fp:
+          if line.startswith('include '):
+            manifest_lines.append(line.split()[-1])
+    except IOError:
+      self.logger.warn('MANIFEST.in not found in %s.\nSkipping user data '
+                       'check.\n', gslib.GSUTIL_DIR)
+      return
+
+    # Look just at top-level directory. We don't try to catch data dropped into
+    # subdirs (like gslib) because that would require deeper parsing of
+    # MANFFEST.in, and most users who drop data into gsutil dir do so at the top
+    # level directory.
+    for filename in os.listdir(gslib.GSUTIL_DIR):
+      if filename.endswith('.pyc'):
+        # Ignore compiled code.
+        continue
+      if filename not in manifest_lines:
+        raise CommandException('\n'.join(textwrap.wrap(
+            'A file (%s) that is not distributed with gsutil was found in '
+            'the gsutil directory. The update command cannot run with user '
+            'data in the gsutil directory.' %
+            os.path.join(gslib.GSUTIL_DIR, filename))))
+
+  def _ExplainIfSudoNeeded(self, tf, dirs_to_remove):
+    """Explains what to do if sudo needed to update gsutil software.
+
+    Happens if gsutil was previously installed by a different user (typically if
+    someone originally installed in a shared file system location, using sudo).
+
+    Args:
+      tf: Opened TarFile.
+      dirs_to_remove: List of directories to remove.
+
+    Raises:
+      CommandException: if errors encountered.
+    """
+    # If running under Windows or Cygwin we don't need (or have) sudo.
+    if IS_CYGWIN or IS_WINDOWS:
+      return
+
+    user_id = os.getuid()
+    if os.stat(gslib.GSUTIL_DIR).st_uid == user_id:
+      return
+
+    # Won't fail - this command runs after main startup code that insists on
+    # having a config file.
+    config_file_list = GetBotoConfigFileList()
+    config_files = ' '.join(config_file_list)
+    self._CleanUpUpdateCommand(tf, dirs_to_remove)
+
+    # Pick current protection of each boto config file for command that restores
+    # protection (rather than fixing at 600) to support use cases like how GCE
+    # installs a service account with an /etc/boto.cfg file protected to 644.
+    chmod_cmds = []
+    for config_file in config_file_list:
+      mode = oct(stat.S_IMODE((os.stat(config_file)[stat.ST_MODE])))
+      chmod_cmds.append('\n\tsudo chmod %s %s' % (mode, config_file))
+
+    raise CommandException('\n'.join(textwrap.wrap(
+        'Since it was installed by a different user previously, you will need '
+        'to update using the following commands. You will be prompted for your '
+        'password, and the install will run as "root". If you\'re unsure what '
+        'this means please ask your system administrator for help:')) + (
+            '\n\tsudo chmod 0644 %s\n\tsudo env BOTO_CONFIG="%s" %s update'
+            '%s') % (config_files, config_files, self.gsutil_path,
+                     ' '.join(chmod_cmds)), informational=True)
+
+  # This list is checked during gsutil update by doing a lowercased
+  # slash-left-stripped check. For example "/Dev" would match the "dev" entry.
+  unsafe_update_dirs = [
+      'applications', 'auto', 'bin', 'boot', 'desktop', 'dev',
+      'documents and settings', 'etc', 'export', 'home', 'kernel', 'lib',
+      'lib32', 'library', 'lost+found', 'mach_kernel', 'media', 'mnt', 'net',
+      'null', 'network', 'opt', 'private', 'proc', 'program files', 'python',
+      'root', 'sbin', 'scripts', 'srv', 'sys', 'system', 'tmp', 'users', 'usr',
+      'var', 'volumes', 'win', 'win32', 'windows', 'winnt',
+  ]
+
+  def _EnsureDirsSafeForUpdate(self, dirs):
+    """Raises Exception if any of dirs is known to be unsafe for gsutil update.
+
+    This provides a fail-safe check to ensure we don't try to overwrite
+    or delete any important directories. (That shouldn't happen given the
+    way we construct tmp dirs, etc., but since the gsutil update cleanup
+    uses shutil.rmtree() it's prudent to add extra checks.)
+
+    Args:
+      dirs: List of directories to check.
+
+    Raises:
+      CommandException: If unsafe directory encountered.
+    """
+    for d in dirs:
+      if not d:
+        d = 'null'
+      if d.lstrip(os.sep).lower() in self.unsafe_update_dirs:
+        raise CommandException('EnsureDirsSafeForUpdate: encountered unsafe '
+                               'directory (%s); aborting update' % d)
+
+  def _CleanUpUpdateCommand(self, tf, dirs_to_remove):
+    """Cleans up temp files etc. from running update command.
+
+    Args:
+      tf: Opened TarFile, or None if none currently open.
+      dirs_to_remove: List of directories to remove.
+
+    """
+    if tf:
+      tf.close()
+    self._EnsureDirsSafeForUpdate(dirs_to_remove)
+    for directory in dirs_to_remove:
+      try:
+        shutil.rmtree(directory)
+      except OSError:
+        # Ignore errors while attempting to remove old dirs under Windows. They
+        # happen because of Windows exclusive file locking, and the update
+        # actually succeeds but just leaves the old versions around in the
+        # user's temp dir.
+        if not IS_WINDOWS:
+          raise
+
+  def RunCommand(self):
+    """Command entry point for the update command."""
+
+    if gslib.IS_PACKAGE_INSTALL:
+      raise CommandException(
+          'The update command is only available for gsutil installed from a '
+          'tarball. If you installed gsutil via another method, use the same '
+          'method to update it.')
+
+    if os.environ.get('CLOUDSDK_WRAPPER') == '1':
+      raise CommandException(
+          'The update command is disabled for Cloud SDK installs. Please run '
+          '"gcloud components update" to update it. Note: the Cloud SDK '
+          'incorporates updates to the underlying tools approximately every 2 '
+          'weeks, so if you are attempting to update to a recently created '
+          'release / pre-release of gsutil it may not yet be available via '
+          'the Cloud SDK.')
+
+    https_validate_certificates = CERTIFICATE_VALIDATION_ENABLED
+    if not https_validate_certificates:
+      raise CommandException(
+          'Your boto configuration has https_validate_certificates = False.\n'
+          'The update command cannot be run this way, for security reasons.')
+
+    self._DisallowUpdataIfDataInGsutilDir()
+
+    force_update = False
+    no_prompt = False
+    if self.sub_opts:
+      for o, unused_a in self.sub_opts:
+        if o == '-f':
+          force_update = True
+        if o == '-n':
+          no_prompt = True
+
+    dirs_to_remove = []
+    tmp_dir = tempfile.mkdtemp()
+    dirs_to_remove.append(tmp_dir)
+    os.chdir(tmp_dir)
+
+    if not no_prompt:
+      self.logger.info('Checking for software update...')
+    if self.args:
+      update_from_url_str = self.args[0]
+      if not update_from_url_str.endswith('.tar.gz'):
+        raise CommandException(
+            'The update command only works with tar.gz files.')
+      for i, result in enumerate(self.WildcardIterator(update_from_url_str)):
+        if i > 0:
+          raise CommandException(
+              'Invalid update URL. Must name a single .tar.gz file.')
+        storage_url = result.storage_url
+        if storage_url.IsFileUrl() and not storage_url.IsDirectory():
+          if not force_update:
+            raise CommandException(
+                ('"update" command does not support "file://" URLs without the '
+                 '-f option.'))
+        elif not (storage_url.IsCloudUrl() and storage_url.IsObject()):
+          raise CommandException(
+              'Invalid update object URL. Must name a single .tar.gz file.')
+    else:
+      update_from_url_str = GSUTIL_PUB_TARBALL
+
+    # Try to retrieve version info from tarball metadata; failing that; download
+    # the tarball and extract the VERSION file. The version lookup will fail
+    # when running the update system test, because it retrieves the tarball from
+    # a temp file rather than a cloud URL (files lack the version metadata).
+    tarball_version = LookUpGsutilVersion(self.gsutil_api, update_from_url_str)
+    if tarball_version:
+      tf = None
+    else:
+      tf = self._FetchAndOpenGsutilTarball(update_from_url_str)
+      tf.extractall()
+      with open(os.path.join('gsutil', 'VERSION'), 'r') as ver_file:
+        tarball_version = ver_file.read().strip()
+
+    if not force_update and gslib.VERSION == tarball_version:
+      self._CleanUpUpdateCommand(tf, dirs_to_remove)
+      if self.args:
+        raise CommandException('You already have %s installed.' %
+                               update_from_url_str, informational=True)
+      else:
+        raise CommandException('You already have the latest gsutil release '
+                               'installed.', informational=True)
+
+    if not no_prompt:
+      (_, major) = CompareVersions(tarball_version, gslib.VERSION)
+      if major:
+        print('\n'.join(textwrap.wrap(
+            'This command will update to the "%s" version of gsutil at %s. '
+            'NOTE: This a major new version, so it is strongly recommended '
+            'that you review the release note details at %s before updating to '
+            'this version, especially if you use gsutil in scripts.'
+            % (tarball_version, gslib.GSUTIL_DIR, RELEASE_NOTES_URL))))
+      else:
+        print('This command will update to the "%s" version of\ngsutil at %s'
+              % (tarball_version, gslib.GSUTIL_DIR))
+    self._ExplainIfSudoNeeded(tf, dirs_to_remove)
+
+    if no_prompt:
+      answer = 'y'
+    else:
+      answer = raw_input('Proceed? [y/N] ')
+    if not answer or answer.lower()[0] != 'y':
+      self._CleanUpUpdateCommand(tf, dirs_to_remove)
+      raise CommandException('Not running update.', informational=True)
+
+    if not tf:
+      tf = self._FetchAndOpenGsutilTarball(update_from_url_str)
+
+    # Ignore keyboard interrupts during the update to reduce the chance someone
+    # hitting ^C leaves gsutil in a broken state.
+    RegisterSignalHandler(signal.SIGINT, signal.SIG_IGN)
+
+    # gslib.GSUTIL_DIR lists the path where the code should end up (like
+    # /usr/local/gsutil), which is one level down from the relative path in the
+    # tarball (since the latter creates files in ./gsutil). So, we need to
+    # extract at the parent directory level.
+    gsutil_bin_parent_dir = os.path.normpath(
+        os.path.join(gslib.GSUTIL_DIR, '..'))
+
+    # Extract tarball to a temporary directory in a sibling to GSUTIL_DIR.
+    old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir)
+    new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir)
+    dirs_to_remove.append(old_dir)
+    dirs_to_remove.append(new_dir)
+    self._EnsureDirsSafeForUpdate(dirs_to_remove)
+    try:
+      tf.extractall(path=new_dir)
+    except Exception, e:
+      self._CleanUpUpdateCommand(tf, dirs_to_remove)
+      raise CommandException('Update failed: %s.' % e)
+
+    # For enterprise mode (shared/central) installation, users with
+    # different user/group than the installation user/group must be
+    # able to run gsutil so we need to do some permissions adjustments
+    # here. Since enterprise mode is not not supported for Windows
+    # users, we can skip this step when running on Windows, which
+    # avoids the problem that Windows has no find or xargs command.
+    if not IS_WINDOWS:
+      # Make all files and dirs in updated area owner-RW and world-R, and make
+      # all directories owner-RWX and world-RX.
+      for dirname, subdirs, filenames in os.walk(new_dir):
+        for filename in filenames:
+          fd = os.open(os.path.join(dirname, filename), os.O_RDONLY)
+          os.fchmod(fd, stat.S_IWRITE | stat.S_IRUSR |
+                    stat.S_IRGRP | stat.S_IROTH)
+          os.close(fd)
+        for subdir in subdirs:
+          fd = os.open(os.path.join(dirname, subdir), os.O_RDONLY)
+          os.fchmod(fd, stat.S_IRWXU | stat.S_IXGRP | stat.S_IXOTH |
+                    stat.S_IRGRP | stat.S_IROTH)
+          os.close(fd)
+
+      # Make main gsutil script owner-RWX and world-RX.
+      fd = os.open(os.path.join(new_dir, 'gsutil', 'gsutil'), os.O_RDONLY)
+      os.fchmod(fd, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP |
+                stat.S_IROTH | stat.S_IXOTH)
+      os.close(fd)
+
+    # Move old installation aside and new into place.
+    os.rename(gslib.GSUTIL_DIR, os.path.join(old_dir, 'old'))
+    os.rename(os.path.join(new_dir, 'gsutil'), gslib.GSUTIL_DIR)
+    self._CleanUpUpdateCommand(tf, dirs_to_remove)
+    RegisterSignalHandler(signal.SIGINT, signal.SIG_DFL)
+    self.logger.info('Update complete.')
+    return 0
+
+  def _FetchAndOpenGsutilTarball(self, update_from_url_str):
+    self.command_runner.RunNamedCommand(
+        'cp', [update_from_url_str, 'file://gsutil.tar.gz'], self.headers,
+        self.debug, skip_update_check=True)
+    # Note: tf is closed in _CleanUpUpdateCommand.
+    tf = tarfile.open('gsutil.tar.gz')
+    tf.errorlevel = 1  # So fatal tarball unpack errors raise exceptions.
+    return tf