Check in the thirdparty libs needed for webkitpy.

Tools/Scripts/webkitpy/common/system/autoinstall.py

Index: Tools/Scripts/webkitpy/common/system/autoinstall.py
diff --git a/Tools/Scripts/webkitpy/common/system/autoinstall.py b/Tools/Scripts/webkitpy/common/system/autoinstall.py
deleted file mode 100644
index 632aa5d7ea72bf6b50ae7b89fd04d7ab6de188fc..0000000000000000000000000000000000000000
--- a/Tools/Scripts/webkitpy/common/system/autoinstall.py
+++ /dev/null
@@ -1,446 +0,0 @@
-# Copyright (c) 2009, Daniel Krech All rights reserved.
-# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org)
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#  * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-#  * Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-#  * Neither the name of the Daniel Krech nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Support for automatically downloading Python packages from an URL."""
-
-
-import codecs
-import logging
-import os
-import shutil
-import stat
-import sys
-import tarfile
-import tempfile
-import urllib
-import urlparse
-import zipfile
-
-_log = logging.getLogger(__name__)
-
-
-class AutoInstaller(object):
-
-    """Supports automatically installing Python packages from an URL.
-
-    Supports uncompressed files, .tar.gz, and .zip formats.
-
-    Basic usage:
-
-    installer = AutoInstaller()
-
-    installer.install(url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b",
-                      url_subpath="pep8-0.5.0/pep8.py")
-    installer.install(url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.2.4.zip",
-                      url_subpath="mechanize")
-
-    """
-
-    def __init__(self, append_to_search_path=False, make_package=True,
-                 target_dir=None, temp_dir=None):
-        """Create an AutoInstaller instance, and set up the target directory.
-
-        Args:
-          append_to_search_path: A boolean value of whether to append the
-                                 target directory to the sys.path search path.
-          make_package: A boolean value of whether to make the target
-                        directory a package.  This adds an __init__.py file
-                        to the target directory -- allowing packages and
-                        modules within the target directory to be imported
-                        explicitly using dotted module names.
-          target_dir: The directory path to which packages should be installed.
-                      Defaults to a subdirectory of the folder containing
-                      this module called "autoinstalled".
-          temp_dir: The directory path to use for any temporary files
-                    generated while downloading, unzipping, and extracting
-                    packages to install.  Defaults to a standard temporary
-                    location generated by the tempfile module.  This
-                    parameter should normally be used only for development
-                    testing.
-
-        """
-        if target_dir is None:
-            this_dir = os.path.dirname(__file__)
-            target_dir = os.path.join(this_dir, "autoinstalled")
-
-        # Ensure that the target directory exists.
-        self._set_up_target_dir(target_dir, append_to_search_path, make_package)
-
-        self._target_dir = target_dir
-        self._temp_dir = temp_dir
-
-    def _write_file(self, path, text, encoding):
-        with codecs.open(path, "w", encoding) as filehandle:
-            filehandle.write(text)
-
-    def _set_up_target_dir(self, target_dir, append_to_search_path,
-                           make_package):
-        """Set up a target directory.
-
-        Args:
-          target_dir: The path to the target directory to set up.
-          append_to_search_path: A boolean value of whether to append the
-                                 target directory to the sys.path search path.
-          make_package: A boolean value of whether to make the target
-                        directory a package.  This adds an __init__.py file
-                        to the target directory -- allowing packages and
-                        modules within the target directory to be imported
-                        explicitly using dotted module names.
-
-        """
-        if not os.path.exists(target_dir):
-            os.makedirs(target_dir)
-
-        if append_to_search_path:
-            sys.path.append(target_dir)
-
-        if make_package:
-            self._make_package(target_dir)
-
-    def _make_package(self, target_dir):
-        init_path = os.path.join(target_dir, "__init__.py")
-        if not os.path.exists(init_path):
-            text = ("# This file is required for Python to search this "
-                    "directory for modules.\n")
-            self._write_file(init_path, text, "ascii")
-
-    def _create_scratch_directory_inner(self, prefix):
-        """Create a scratch directory without exception handling.
-
-        Creates a scratch directory inside the AutoInstaller temp
-        directory self._temp_dir, or inside a platform-dependent temp
-        directory if self._temp_dir is None.  Returns the path to the
-        created scratch directory.
-
-        Raises:
-          OSError: [Errno 2] if the containing temp directory self._temp_dir
-                             is not None and does not exist.
-
-        """
-        # The tempfile.mkdtemp() method function requires that the
-        # directory corresponding to the "dir" parameter already exist
-        # if it is not None.
-        scratch_dir = tempfile.mkdtemp(prefix=prefix.replace('/', '.'), dir=self._temp_dir)
-        return scratch_dir
-
-    def _create_scratch_directory(self, target_name):
-        """Create a temporary scratch directory, and return its path.
-
-        The scratch directory is generated inside the temp directory
-        of this AutoInstaller instance.  This method also creates the
-        temp directory if it does not already exist.
-
-        """
-        prefix = target_name.replace(os.sep, "_") + "_"
-        try:
-            scratch_dir = self._create_scratch_directory_inner(prefix)
-        except OSError:
-            # Handle case of containing temp directory not existing--
-            # OSError: [Errno 2] No such file or directory:...
-            temp_dir = self._temp_dir
-            if temp_dir is None or os.path.exists(temp_dir):
-                raise
-            # Else try again after creating the temp directory.
-            os.makedirs(temp_dir)
-            scratch_dir = self._create_scratch_directory_inner(prefix)
-
-        return scratch_dir
-
-    def _url_downloaded_path(self, target_name):
-        return os.path.join(self._target_dir, ".%s.url" % target_name.replace('/', '_'))
-
-    def _is_downloaded(self, target_name, url):
-        version_path = self._url_downloaded_path(target_name)
-
-        if not os.path.exists(version_path):
-            return False
-
-        with codecs.open(version_path, "r", "utf-8") as filehandle:
-            return filehandle.read().strip() == url.strip()
-
-    def _record_url_downloaded(self, target_name, url):
-        version_path = self._url_downloaded_path(target_name)
-        self._write_file(version_path, url, "utf-8")
-
-    def _extract_targz(self, path, scratch_dir):
-        # tarfile.extractall() extracts to a path without the trailing ".tar.gz".
-        target_basename = os.path.basename(path[:-len(".tar.gz")])
-        target_path = os.path.join(scratch_dir, target_basename)
-
-        try:
-            tar_file = tarfile.open(path)
-        except tarfile.ReadError, err:
-            # Append existing Error message to new Error.
-            message = ("Could not open tar file: %s\n"
-                       " The file probably does not have the correct format.\n"
-                       " --> Inner message: %s"
-                       % (path, err))
-            raise Exception(message)
-
-        try:
-            tar_file.extractall(target_path)
-        finally:
-            tar_file.close()
-
-        return target_path
-
-    # This is a replacement for ZipFile.extractall(), which is
-    # available in Python 2.6 but not in earlier versions.
-    # NOTE: The version in 2.6.1 (which shipped on Snow Leopard) is broken!
-    def _extract_all(self, zip_file, target_dir):
-        for name in zip_file.namelist():
-            path = os.path.join(target_dir, name)
-            if not os.path.basename(path):
-                # Then the path ends in a slash, so it is a directory.
-                os.makedirs(path)
-                continue
-
-            try:
-                # We open this file w/o encoding, as we're reading/writing
-                # the raw byte-stream from the zip file.
-                outfile = open(path, 'wb')
-            except IOError:
-                # Not all zip files seem to list the directories explicitly,
-                # so try again after creating the containing directory.
-                _log.debug("Got IOError: retrying after creating directory...")
-                dirname = os.path.dirname(path)
-                os.makedirs(dirname)
-                outfile = open(path, 'wb')
-
-            try:
-                outfile.write(zip_file.read(name))
-            finally:
-                outfile.close()
-
-    def _unzip(self, path, scratch_dir):
-        # zipfile.extractall() extracts to a path without the trailing ".zip".
-        target_basename = os.path.basename(path[:-len(".zip")])
-        target_path = os.path.join(scratch_dir, target_basename)
-
-        try:
-            zip_file = zipfile.ZipFile(path, "r")
-        except zipfile.BadZipfile, err:
-            message = ("Could not open zip file: %s\n"
-                       " --> Inner message: %s"
-                       % (path, err))
-            raise Exception(message)
-
-        try:
-            self._extract_all(zip_file, scratch_dir)
-        finally:
-            zip_file.close()
-
-        return target_path
-
-    def _prepare_package(self, path, scratch_dir):
-        """Prepare a package for use, if necessary, and return the new path.
-
-        For example, this method unzips zipped files and extracts
-        tar files.
-
-        Args:
-          path: The path to the downloaded URL contents.
-          scratch_dir: The scratch directory.  Note that the scratch
-                       directory contains the file designated by the
-                       path parameter.
-
-        """
-        # FIXME: Add other natural extensions.
-        if path.endswith(".zip"):
-            new_path = self._unzip(path, scratch_dir)
-        elif path.endswith(".tar.gz"):
-            new_path = self._extract_targz(path, scratch_dir)
-        else:
-            # No preparation is needed.
-            new_path = path
-
-        return new_path
-
-    def _download_to_stream(self, url, stream):
-        failures = 0
-        while True:
-            try:
-                netstream = urllib.urlopen(url)
-                break
-            except IOError, err:
-                # Try multiple times
-                if failures < 5:
-                    _log.warning("Failed to download %s, %s retrying" % (
-                        url, err))
-                    failures += 1
-                    continue
-
-                # Append existing Error message to new Error.
-                message = ('Could not download Python modules from URL "%s".\n'
-                           " Make sure you are connected to the internet.\n"
-                           " You must be connected to the internet when "
-                           "downloading needed modules for the first time.\n"
-                           " --> Inner message: %s"
-                           % (url, err))
-                raise IOError(message)
-        code = 200
-        if hasattr(netstream, "getcode"):
-            code = netstream.getcode()
-        if not 200 <= code < 300:
-            raise ValueError("HTTP Error code %s" % code)
-
-        BUFSIZE = 2**13  # 8KB
-        while True:
-            data = netstream.read(BUFSIZE)
-            if not data:
-                break
-            stream.write(data)
-        netstream.close()
-
-    def _download(self, url, scratch_dir):
-        url_path = urlparse.urlsplit(url)[2]
-        url_path = os.path.normpath(url_path)  # Removes trailing slash.
-        target_filename = os.path.basename(url_path)
-        target_path = os.path.join(scratch_dir, target_filename)
-
-        should_clear_proxy = False
-        with open(target_path, "wb") as stream:
-            try:
-                # FIXME: This is a workaround for a weird curl limitation we're hitting downloading from pypi.
-                # We can delete this once we can check the libs directly into third_party and get rid of all of
-                # this code. See crbug.com/230706 .
-                if 'HTTPS_PROXY' in os.environ and not 'https_proxy' in os.environ:
-                    os.environ['https_proxy'] = os.environ['HTTPS_PROXY']
-                    should_clear_proxy = True
-
-                self._download_to_stream(url, stream)
-
-            finally:
-                if should_clear_proxy:
-                    del os.environ['https_proxy']
-
-        return target_path
-
-    def _install(self, scratch_dir, package_name, target_path, url, url_subpath, files_to_remove):
-        """Install a python package from an URL.
-
-        This internal method overwrites the target path if the target
-        path already exists.
-
-        """
-        path = self._download(url=url, scratch_dir=scratch_dir)
-        path = self._prepare_package(path, scratch_dir)
-
-        if url_subpath is None:
-            source_path = path
-        else:
-            source_path = os.path.join(path, url_subpath)
-
-        for filename in files_to_remove:
-            path = os.path.join(source_path, filename.replace('/', os.sep))
-            if os.path.exists(path):
-                # Pre-emptively change the permissions to #0777 to try and work around win32 permissions issues.
-                os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
-                os.remove(path)
-
-        if os.path.exists(target_path):
-            if os.path.isdir(target_path):
-                shutil.rmtree(target_path, ignore_errors=True)
-            else:
-                os.remove(target_path)
-
-        # shutil.move() command creates intermediate directories if they do not exist.
-        shutil.move(source_path, target_path)
-
-        # ensure all the new directories are importable.
-        intermediate_dirs = os.path.dirname(os.path.relpath(target_path, self._target_dir))
-        parent_dirname = self._target_dir
-        for dirname in intermediate_dirs.split(os.sep):
-            parent_dirname = os.path.join(parent_dirname, dirname)
-            self._make_package(parent_dirname)
-
-        self._record_url_downloaded(package_name, url)
-
-    def install(self, url, should_refresh=False, target_name=None,
-                url_subpath=None, files_to_remove=None):
-        """Install a python package from an URL.
-
-        Args:
-          url: The URL from which to download the package.
-
-        Optional Args:
-          should_refresh: A boolean value of whether the package should be
-                          downloaded again if the package is already present.
-          target_name: The name of the folder or file in the autoinstaller
-                       target directory at which the package should be
-                       installed.  Defaults to the base name of the
-                       URL sub-path.  This parameter must be provided if
-                       the URL sub-path is not specified.
-          url_subpath: The relative path of the URL directory that should
-                       be installed.  Defaults to the full directory, or
-                       the entire URL contents.
-
-        """
-        if target_name is None:
-            if not url_subpath:
-                raise ValueError('The "target_name" parameter must be '
-                                 'provided if the "url_subpath" parameter '
-                                 "is not provided.")
-            # Remove any trailing slashes.
-            url_subpath = os.path.normpath(url_subpath)
-            target_name = os.path.basename(url_subpath)
-
-        target_path = os.path.join(self._target_dir, target_name.replace('/', os.sep))
-        if not should_refresh and self._is_downloaded(target_name, url):
-            return False
-
-        files_to_remove = files_to_remove or []
-        package_name = target_name.replace(os.sep, '.')
-        _log.info("Auto-installing package: %s" % package_name)
-
-        # The scratch directory is where we will download and prepare
-        # files specific to this install until they are ready to move
-        # into place.
-        scratch_dir = self._create_scratch_directory(target_name)
-
-        try:
-            self._install(package_name=package_name,
-                          target_path=target_path,
-                          scratch_dir=scratch_dir,
-                          url=url,
-                          url_subpath=url_subpath,
-                          files_to_remove=files_to_remove)
-        except Exception, err:
-            # Append existing Error message to new Error.
-            message = ("Error auto-installing the %s package to:\n"
-                       ' "%s"\n'
-                       " --> Inner message: %s"
-                       % (target_name, target_path, err))
-            raise Exception(message)
-        finally:
-            shutil.rmtree(scratch_dir, ignore_errors=True)
-        _log.debug('Auto-installed %s to:' % url)
-        _log.debug('    "%s"' % target_path)
-        return True