tool for repackaging chrome.perf builds for manual bisect

tools/bisect_repackage/bisect_repackage_utils.py

+# Copyright (c) 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" Set of basic operations/utilities that are used by repacakging builds
+for bisect """
+
+import os
+import ast
+import base64
+import cStringIO
+import copy
+import errno
+import fnmatch
+import glob
+import math
+import multiprocessing
+import os
+import re
+import shutil
+import socket
+import stat
+import string  # pylint: disable=W0402
+import subprocess
+import sys
+import threading
+import time
+import traceback
+import urllib
+import zipfile
+import zlib
+
+CREDENTIAL_ERROR_MESSAGE = ('You are attempting to access protected data with '
+                            'no configured credentials')
+
+class ExternalError(Exception):
+  pass
+def IsWindows():
+  return sys.platform == 'cygwin' or sys.platform.startswith('win')
+
+def IsLinux():
+  return sys.platform.startswith('linux')
+
+def IsMac():
+  return sys.platform.startswith('darwin')
+
+WIN_LINK_FUNC = None
+
+try:
+  if sys.platform.startswith('win'):
+    import ctypes
+    # There's 4 possibilities on Windows for links:
+    # 1. Symbolic file links;
+    # 2. Symbolic directory links;
+    # 3. Hardlinked files;
+    # 4. Junctioned directories.
+    # (Hardlinked directories don't really exist.)
+    #
+    # 7-Zip does not handle symbolic file links as we want (it puts the
+    # content of the link, not what it refers to, and reports "CRC Error" on
+    # extraction). It does work as expected for symbolic directory links.
+    # Because the majority of the large files are in the root of the staging
+    # directory, we do however need to handle file links, so we do this with
+    # hardlinking. Junctioning requires a huge whack of code, so we take the
+    # slightly odd tactic of using #2 and #3, but not #1 and #4. That is,
+    # hardlinks for files, but symbolic links for directories.
+    def _WIN_LINK_FUNC(src, dst):
+      print 'linking %s -> %s' % (src, dst)
+      if os.path.isdir(src):
+        if not ctypes.windll.kernel32.CreateSymbolicLinkA(
+            str(dst), str(os.path.abspath(src)), 1):
+          raise ctypes.WinError()
+      else:
+        if not ctypes.windll.kernel32.CreateHardLinkA(str(dst), str(src), 0):
+          raise ctypes.WinError()
+    WIN_LINK_FUNC = _WIN_LINK_FUNC
+except ImportError:
+  # If we don't have ctypes or aren't on Windows, leave WIN_LINK_FUNC as None.
+  pass
+
+
+
+class PathNotFound(Exception):
+  pass
+
+def MaybeMakeDirectory(*path):

[dimu, 2016/08/19 04:14:05]

better naming

+  """Creates an entire path, if it doesn't already exist."""
+  file_path = os.path.join(*path)
+  try:
+    os.makedirs(file_path)
+  except OSError, e:
+    if e.errno != errno.EEXIST:
+      raise
+
+def RemovePath(*path):
+  """Removes the file or directory at 'path', if it exists."""
+  file_path = os.path.join(*path)
+  if os.path.exists(file_path):
+    if os.path.isdir(file_path):
+      RemoveDirectory(file_path)
+    else:
+      RemoveFile(file_path)
+
+def MoveFile(path, new_path):
+  """Moves the file located at 'path' to 'new_path', if it exists."""
+  try:
+    RemoveFile(new_path)
+    os.rename(path, new_path)
+  except OSError, e:
+    if e.errno != errno.ENOENT:
+      raise
+
+def RemoveFile(*path):
+  """Removes the file located at 'path', if it exists."""
+  file_path = os.path.join(*path)
+  try:
+    os.remove(file_path)
+  except OSError, e:
+    if e.errno != errno.ENOENT:
+      raise
+
+def CheckDepotToolsInPath():
+  delimiter = ';' if sys.platform.startswith('win') else ':'
+  path_list = os.environ['PATH'].split(delimiter)
+  for path in path_list:
+    if path.rstrip(os.path.sep).endswith('depot_tools'):
+      return path
+  return None
+
+def RunGsutilCommand(args):
+  gsutil_path = CheckDepotToolsInPath()
+  if gsutil_path is None:
+    print ('Follow the instructions in this document '
+           'http://dev.chromium.org/developers/how-tos/install-depot-tools'
+           ' to install depot_tools and then try again.')
+    sys.exit(1)
+  gsutil_path = os.path.join(gsutil_path, 'third_party', 'gsutil', 'gsutil')
+  gsutil = subprocess.Popen([sys.executable, gsutil_path] + args,
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                            env=None)
+  stdout, stderr = gsutil.communicate()
+  if gsutil.returncode:
+    if (re.findall(r'status[ |=]40[1|3]', stderr) or
+        stderr.startswith(CREDENTIAL_ERROR_MESSAGE)):
+      print ('Follow these steps to configure your credentials and try'
+             ' running the bisect-builds.py again.:\n'
+             '  1. Run "python %s config" and follow its instructions.\n'
+             '  2. If you have a @google.com account, use that account.\n'
+             '  3. For the project-id, just enter 0.' % gsutil_path)
+      sys.exit(1)
+    else:
+      raise Exception('Error running the gsutil command: %s' % stderr)
+  return stdout
+
+def GSutilList(bucket):
+  query = '%s/' %(bucket)
+  stdout = RunGsutilCommand(['ls', query])
+  return [url[len(query):].strip('/') for url in stdout.splitlines()]
+
+def GSUtilDownloadFile(src, dst):
+  command = ['cp', src, dst]
+  return RunGsutilCommand(command)
+
+def GSUtilCopy(source, dest):
+  if not source.startswith('gs://') and not source.startswith('file://'):
+    source = 'file://' + source
+  if not dest.startswith('gs://') and not dest.startswith('file://'):
+    dest = 'file://' + dest
+  command = ['cp']
+  command.extend([source, dest])
+  return RunGsutilCommand(command)
+
+def RunCommand(cmd, cwd=None):
+  """Runs the given command and returns the exit code.
+
+  Args:
+    cmd: list of command arguments.
+    cwd: working directory to execute the command, or None if the current
+         working directory should be used.
+
+  Returns:
+    The exit code of the command.
+  """
+  process = subprocess.Popen(cmd, cwd=cwd)
+  process.wait()
+  return process.returncode
+
+
+
+def CopyFileToDir(src_path, dest_dir, dest_fn=None, link_ok=False):
+  """Copies the file found at src_path to the dest_dir directory, with metadata.
+
+  If dest_fn is specified, the src_path is copied to that name in dest_dir,
+  otherwise it is copied to a file of the same name.
+
+  Raises PathNotFound if either the file or the directory is not found.
+  """
+  # Verify the file and directory separately so we can tell them apart and
+  # raise PathNotFound rather than shutil.copyfile's IOError.
+  if not os.path.isfile(src_path):
+    raise PathNotFound('Unable to find file %s' % src_path)
+  if not os.path.isdir(dest_dir):
+    raise PathNotFound('Unable to find dir %s' % dest_dir)
+  src_file = os.path.basename(src_path)
+  if dest_fn:
+    # If we have ctypes and the caller doesn't mind links, use that to
+    # try to make the copy faster on Windows. http://crbug.com/418702.
+    if link_ok and WIN_LINK_FUNC:
+      WIN_LINK_FUNC(src_path, os.path.join(dest_dir, dest_fn))
+    else:
+      shutil.copy2(src_path, os.path.join(dest_dir, dest_fn))
+  else:
+    shutil.copy2(src_path, os.path.join(dest_dir, src_file))
+
+def RemoveDirectory(*path):
+  """Recursively removes a directory, even if it's marked read-only.
+
+  Remove the directory located at *path, if it exists.
+
+  shutil.rmtree() doesn't work on Windows if any of the files or directories
+  are read-only, which svn repositories and some .svn files are.  We need to
+  be able to force the files to be writable (i.e., deletable) as we traverse
+  the tree.
+
+  Even with all this, Windows still sometimes fails to delete a file, citing
+  a permission error (maybe something to do with antivirus scans or disk
+  indexing).  The best suggestion any of the user forums had was to wait a
+  bit and try again, so we do that too.  It's hand-waving, but sometimes it
+  works. :/
+  """
+  file_path = os.path.join(*path)
+  if not os.path.exists(file_path):
+    return
+
+  if sys.platform == 'win32':
+    # Give up and use cmd.exe's rd command.
+    file_path = os.path.normcase(file_path)
+    for _ in xrange(3):
+      print 'RemoveDirectory running %s' % (' '.join(
+          ['cmd.exe', '/c', 'rd', '/q', '/s', file_path]))
+      if not subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', file_path]):
+        break
+      print '  Failed'
+      time.sleep(3)
+    return
+
+  def RemoveWithRetry_non_win(rmfunc, path):
+    if os.path.islink(path):
+      return os.remove(path)
+    else:
+      return rmfunc(path)
+
+  remove_with_retry = RemoveWithRetry_non_win
+
+  def RmTreeOnError(function, path, excinfo):
+    r"""This works around a problem whereby python 2.x on Windows has no ability
+    to check for symbolic links.  os.path.islink always returns False.  But
+    shutil.rmtree will fail if invoked on a symbolic link whose target was
+    deleted before the link.  E.g., reproduce like this:
+    > mkdir test
+    > mkdir test\1
+    > mklink /D test\current test\1
+    > python -c "import chromium_utils; chromium_utils.RemoveDirectory('test')"
+    To avoid this issue, we pass this error-handling function to rmtree.  If
+    we see the exact sort of failure, we ignore it.  All other failures we re-
+    raise.
+    """
+
+    exception_type = excinfo[0]
+    exception_value = excinfo[1]
+    # If shutil.rmtree encounters a symbolic link on Windows, os.listdir will
+    # fail with a WindowsError exception with an ENOENT errno (i.e., file not
+    # found).  We'll ignore that error.  Note that WindowsError is not defined
+    # for non-Windows platforms, so we use OSError (of which it is a subclass)
+    # to avoid lint complaints about an undefined global on non-Windows
+    # platforms.
+    if (function is os.listdir) and issubclass(exception_type, OSError):
+      if exception_value.errno == errno.ENOENT:
+        # File does not exist, and we're trying to delete, so we can ignore the
+        # failure.
+        print 'WARNING:  Failed to list %s during rmtree.  Ignoring.\n' % path
+      else:
+        raise
+    else:
+      raise
+
+  for root, dirs, files in os.walk(file_path, topdown=False):
+    # For POSIX:  making the directory writable guarantees removability.
+    # Windows will ignore the non-read-only bits in the chmod value.
+    os.chmod(root, 0770)
+    for name in files:
+      remove_with_retry(os.remove, os.path.join(root, name))
+    for name in dirs:
+      remove_with_retry(lambda p: shutil.rmtree(p, onerror=RmTreeOnError),
+                        os.path.join(root, name))
+
+  remove_with_retry(os.rmdir, file_path)
+
+
+
+def MakeZip(output_dir, archive_name, file_list, file_relative_dir,
+            raise_error=True, remove_archive_directory=True, strip_files=None):
+  """Packs files into a new zip archive.
+
+  Files are first copied into a directory within the output_dir named for
+  the archive_name, which will be created if necessary and emptied if it
+  already exists.  The files are then then packed using archive names
+  relative to the output_dir.  That is, if the zipfile is unpacked in place,
+  it will create a directory identical to the new archive_name directory, in
+  the output_dir.  The zip file will be named as the archive_name, plus
+  '.zip'.
+
+  Args:
+    output_dir: Absolute path to the directory in which the archive is to
+      be created.
+    archive_dir: Subdirectory of output_dir holding files to be added to
+      the new zipfile.
+    file_list: List of paths to files or subdirectories, relative to the
+      file_relative_dir.
+    file_relative_dir: Absolute path to the directory containing the files
+      and subdirectories in the file_list.
+    raise_error: Whether to raise a PathNotFound error if one of the files in
+      the list is not found.
+    remove_archive_directory: Whether to remove the archive staging directory
+      before copying files over to it.
+    strip_files: List of executable files to strip symbols when zipping
+
+  Returns:
+    A tuple consisting of (archive_dir, zip_file_path), where archive_dir
+    is the full path to the newly created archive_name subdirectory.
+
+  Raises:
+    PathNotFound if any of the files in the list is not found, unless
+    raise_error is False, in which case the error will be ignored.
+  """
+
+  start_time = time.clock()
+  # Collect files into the archive directory.
+  archive_dir = os.path.join(output_dir, archive_name)
+  print 'output_dir: %s, archive_name: %s' % (output_dir, archive_name)
+  print 'archive_dir: %s, remove_archive_directory: %s, exists: %s' % (
+      archive_dir, remove_archive_directory, os.path.exists(archive_dir))
+  if remove_archive_directory and os.path.exists(archive_dir):
+    # Move it even if it's not a directory as expected. This can happen with
+    # FILES.cfg archive creation where we create an archive staging directory
+    # that is the same name as the ultimate archive name.
+    if not os.path.isdir(archive_dir):
+      print 'Moving old "%s" file to create same name directory.' % archive_dir
+      previous_archive_file = '%s.old' % archive_dir
+      MoveFile(archive_dir, previous_archive_file)
+    else:
+      print 'Removing %s' % archive_dir
+      RemoveDirectory(archive_dir)
+      print 'Now, os.path.exists(%s): %s' % (
+          archive_dir, os.path.exists(archive_dir))
+  MaybeMakeDirectory(archive_dir)
+  for needed_file in file_list:
+    needed_file = needed_file.rstrip()
+    # These paths are relative to the file_relative_dir.  We need to copy
+    # them over maintaining the relative directories, where applicable.
+    src_path = os.path.join(file_relative_dir, needed_file)
+    dirname, basename = os.path.split(needed_file)
+    try:
+      if os.path.isdir(src_path):
+        if WIN_LINK_FUNC:
+          WIN_LINK_FUNC(src_path, os.path.join(archive_dir, needed_file))
+        else:
+          shutil.copytree(src_path, os.path.join(archive_dir, needed_file),
+                          symlinks=True)
+      elif dirname != '' and basename != '':
+        dest_dir = os.path.join(archive_dir, dirname)
+        MaybeMakeDirectory(dest_dir)
+        CopyFileToDir(src_path, dest_dir, basename, link_ok=True)
+        if strip_files and basename in strip_files:
+          cmd = ['strip', os.path.join(dest_dir, basename)]
+          RunCommand(cmd)
+      else:
+        CopyFileToDir(src_path, archive_dir, basename, link_ok=True)
+        if strip_files and basename in strip_files:
+          cmd = ['strip', os.path.join(archive_dir, basename)]
+          RunCommand(cmd)
+    except PathNotFound:
+      if raise_error:
+        raise
+  end_time = time.clock()
+  print 'Took %f seconds to create archive directory.' % (end_time - start_time)
+
+  # Pack the zip file.
+  output_file = '%s.zip' % archive_dir
+  previous_file = '%s_old.zip' % archive_dir
+  MoveFile(output_file, previous_file)
+
+  # If we have 7z, use that as it's much faster. See http://crbug.com/418702.
+  windows_zip_cmd = None
+  if os.path.exists('C:\\Program Files\\7-Zip\\7z.exe'):
+    windows_zip_cmd = ['C:\\Program Files\\7-Zip\\7z.exe', 'a', '-y', '-mx1']
+
+  # On Windows we use the python zip module; on Linux and Mac, we use the zip
+  # command as it will handle links and file bits (executable).  Which is much
+  # easier then trying to do that with ZipInfo options.
+  start_time = time.clock()
+  if IsWindows() and not windows_zip_cmd:
+    print 'Creating %s' % output_file
+
+    def _Addfiles(to_zip_file, dirname, files_to_add):
+      for this_file in files_to_add:
+        archive_name = this_file
+        this_path = os.path.join(dirname, this_file)
+        if os.path.isfile(this_path):
+          # Store files named relative to the outer output_dir.
+          archive_name = this_path.replace(output_dir + os.sep, '')
+          if os.path.getsize(this_path) == 0:
+            compress_method = zipfile.ZIP_STORED
+          else:
+            compress_method = zipfile.ZIP_DEFLATED
+          to_zip_file.write(this_path, archive_name, compress_method)
+          print 'Adding %s' % archive_name
+    zip_file = zipfile.ZipFile(output_file, 'w', zipfile.ZIP_DEFLATED,
+                               allowZip64=True)
+    try:
+      os.path.walk(archive_dir, _Addfiles, zip_file)
+    finally:
+      zip_file.close()
+  else:
+    if IsMac() or IsLinux():
+      zip_cmd = ['zip', '-yr1']
+    else:
+      zip_cmd = windows_zip_cmd
+    saved_dir = os.getcwd()
+    os.chdir(os.path.dirname(archive_dir))
+    command = zip_cmd + [output_file, os.path.basename(archive_dir)]
+    result = RunCommand(command)
+    os.chdir(saved_dir)
+    if result and raise_error:
+      raise ExternalError('zip failed: %s => %s' %
+                          (str(command), result))
+  end_time = time.clock()
+  print 'Took %f seconds to create zip.' % (end_time - start_time)
+  return (archive_dir, output_file)
+
+
+def ExtractZip(filename, output_dir, verbose=True):
+  """ Extract the zip archive in the output directory.
+  """
+  MaybeMakeDirectory(output_dir)
+
+  # On Linux and Mac, we use the unzip command as it will
+  # handle links and file bits (executable), which is much
+  # easier then trying to do that with ZipInfo options.
+  #
+  # The Mac Version of unzip unfortunately does not support Zip64, whereas
+  # the python module does, so we have to fallback to the python zip module
+  # on Mac if the filesize is greater than 4GB.
+  #
+  # On Windows, try to use 7z if it is installed, otherwise fall back to python
+  # zip module and pray we don't have files larger than 512MB to unzip.
+  unzip_cmd = None
+  if ((IsMac() and os.path.getsize(filename) < 4 * 1024 * 1024 * 1024)
+      or IsLinux()):
+    unzip_cmd = ['unzip', '-o']
+  elif IsWindows() and os.path.exists('C:\\Program Files\\7-Zip\\7z.exe'):
+    unzip_cmd = ['C:\\Program Files\\7-Zip\\7z.exe', 'x', '-y']
+
+  if unzip_cmd:
+    # Make sure path is absolute before changing directories.
+    filepath = os.path.abspath(filename)
+    saved_dir = os.getcwd()
+    os.chdir(output_dir)
+    command = unzip_cmd + [filepath]
+    result = RunCommand(command)
+    os.chdir(saved_dir)
+    if result:
+      raise ExternalError('unzip failed: %s => %s' % (str(command), result))
+  else:
+    assert IsWindows() or IsMac()
+    zf = zipfile.ZipFile(filename)
+    # TODO(hinoka): This can be multiprocessed.
+    for name in zf.namelist():
+      if verbose:
+        print 'Extracting %s' % name
+      zf.extract(name, output_dir)
+      if IsMac():
+        # Restore permission bits.
+        os.chmod(os.path.join(output_dir, name),
+                 zf.getinfo(name).external_attr >> 16L)