Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(20)

Unified Diff: tools/auto_bisect/fetch_build.py

Issue 548233002: Add a module to fetch builds from different types of builders. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Remove print in run_tests and move changes to sys.path into main. Created 6 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « tools/auto_bisect/bisect_utils.py ('k') | tools/auto_bisect/fetch_build_test.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: tools/auto_bisect/fetch_build.py
diff --git a/tools/auto_bisect/fetch_build.py b/tools/auto_bisect/fetch_build.py
new file mode 100644
index 0000000000000000000000000000000000000000..711cb01cf9bc3ea13a7693dc98dd151af2d29ee6
--- /dev/null
+++ b/tools/auto_bisect/fetch_build.py
@@ -0,0 +1,357 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This module contains functions for fetching and extracting archived builds.
+
+The builds may be stored in different places by different types of builders;
+for example, builders on tryserver.chromium.perf stores builds in one place,
+while builders on chromium.linux store builds in another.
+
+This module can be either imported or run as a stand-alone script to download
+and extract a build.
+
+Usage: fetch_build.py <type> <revision> <output_dir> [options]
+"""
+
+import argparse
+import errno
+import os
+import shutil
+import sys
+import zipfile
+
+# Telemetry (src/tools/telemetry) is expected to be in the PYTHONPATH.
+from telemetry.util import cloud_storage
+
+import bisect_utils
+
+# Possible builder types.
+PERF_BUILDER = 'perf'
+FULL_BUILDER = 'full'
+
+
+def FetchBuild(builder_type, revision, output_dir, target_arch='ia32',
+ target_platform='chromium', deps_patch_sha=None):
+ """Downloads and extracts a build for a particular revision.
+
+ If the build is successfully downloaded and extracted to |output_dir|, the
+ downloaded archive file is also deleted.
+
+ Args:
+ revision: Revision string, e.g. a git commit hash or SVN revision.
+ builder_type: Type of build archive.
+ target_arch: Architecture, e.g. "ia32".
+ target_platform: Platform name, e.g. "chromium" or "android".
+ deps_patch_sha: SHA1 hash of a DEPS file, if we want to fetch a build for
+ a Chromium revision with custom dependencies.
+
+ Raises:
+ IOError: Unzipping failed.
+ OSError: Directory creation or deletion failed.
+ """
+ build_archive = BuildArchive.Create(
+ builder_type, target_arch=target_arch, target_platform=target_platform)
+ bucket = build_archive.BucketName()
+ remote_path = build_archive.FilePath(revision, deps_patch_sha=deps_patch_sha)
+
+ filename = FetchFromCloudStorage(bucket, remote_path, output_dir)
+ if not filename:
+ raise RuntimeError('Failed to fetch gs://%s/%s.' % (bucket, remote_path))
+
+ Unzip(filename, output_dir)
+
+ if os.path.exists(filename):
+ os.remove(filename)
+
+
+class BuildArchive(object):
+ """Represents a place where builds of some type are stored.
+
+ There are two pieces of information required to locate a file in Google
+ Cloud Storage, bucket name and file path. Subclasses of this class contain
+ specific logic about which bucket names and paths should be used to fetch
+ a build.
+ """
+
+ @staticmethod
+ def Create(builder_type, target_arch='ia32', target_platform='chromium'):
+ if builder_type == PERF_BUILDER:
+ return PerfBuildArchive(target_arch, target_platform)
+ if builder_type == FULL_BUILDER:
+ return FullBuildArchive(target_arch, target_platform)
+ raise NotImplementedError('Builder type "%s" not supported.' % builder_type)
+
+ def __init__(self, target_arch='ia32', target_platform='chromium'):
+ if bisect_utils.IsLinuxHost() and target_platform == 'android':
+ self._platform = 'android'
+ elif bisect_utils.IsLinuxHost():
+ self._platform = 'linux'
+ elif bisect_utils.IsMacHost():
+ self._platform = 'mac'
+ elif bisect_utils.Is64BitWindows() and target_arch == 'x64':
+ self._platform = 'win64'
+ elif bisect_utils.IsWindowsHost():
+ self._platform = 'win'
+ else:
+ raise NotImplementedError('Unknown platform "%s".' % sys.platform)
+
+ def BucketName(self):
+ raise NotImplementedError()
+
+ def FilePath(self, revision, deps_patch_sha=None):
+ """Returns the remote file path to download a build from.
+
+ Args:
+ revision: A Chromium revision; this could be a git commit hash or
+ commit position or SVN revision number.
+ deps_patch_sha: The SHA1 hash of a patch to the DEPS file, which
+ uniquely identifies a change to use a particular revision of
+ a dependency.
+
+ Returns:
+ A file path, which not does not include a bucket name.
+ """
+ raise NotImplementedError()
+
+ def _ZipFileName(self, revision, deps_patch_sha=None):
+ """Gets the file name of a zip archive for a particular revision.
+
+ This returns a file name of the form full-build-<platform>_<revision>.zip,
+ which is a format used by multiple types of builders that store archives.
+
+ Args:
+ revision: A git commit hash or other revision string.
+ deps_patch_sha: SHA1 hash of a DEPS file patch.
+
+ Returns:
+ The archive file name.
+ """
+ base_name = 'full-build-%s' % self._PlatformName()
+ if deps_patch_sha:
+ revision = '%s_%s' % (revision , deps_patch_sha)
+ return '%s_%s.zip' % (base_name, revision)
+
+ def _PlatformName(self):
+ """Return a string to be used in paths for the platform."""
+ if self._platform in ('win', 'win64'):
+ # Build archive for win64 is still stored with "win32" in the name.
+ return 'win32'
+ if self._platform in ('linux', 'android'):
+ # Android builds are also stored with "linux" in the name.
+ return 'linux'
+ if self._platform == 'mac':
+ return 'mac'
+ raise NotImplementedError('Unknown platform "%s".' % sys.platform)
+
+
+class PerfBuildArchive(BuildArchive):
+
+ def BucketName(self):
+ return 'chrome-perf'
+
+ def FilePath(self, revision, deps_patch_sha=None):
+ return '%s/%s' % (self._ArchiveDirectory(),
+ self._ZipFileName(revision, deps_patch_sha))
+
+ def _ArchiveDirectory(self):
+ """Returns the directory name to download builds from."""
+ platform_to_directory = {
+ 'android': 'android_perf_rel',
+ 'linux': 'Linux Builder',
+ 'mac': 'Mac Builder',
+ 'win64': 'Win x64 Builder',
+ 'win': 'Win Builder',
+ }
+ assert self._platform in platform_to_directory
+ return platform_to_directory.get(self._platform)
+
+
+class FullBuildArchive(BuildArchive):
+
+ def BucketName(self):
+ platform_to_bucket = {
+ 'android': 'chromium-android',
+ 'linux': 'chromium-linux-archive',
+ 'mac': 'chromium-mac-archive',
+ 'win64': 'chromium-win-archive',
+ 'win': 'chromium-win-archive',
+ }
+ assert self._platform in platform_to_bucket
+ return platform_to_bucket.get(self._platform)
+
+ def FilePath(self, revision, deps_patch_sha=None):
+ return '%s/%s' % (self._ArchiveDirectory(),
+ self._ZipFileName(revision, deps_patch_sha))
+
+ def _ArchiveDirectory(self):
+ """Returns the remote directory to download builds from."""
+ platform_to_directory = {
+ 'android': 'android_main_rel',
+ 'linux': 'chromium.linux/Linux Builder',
+ 'mac': 'chromium.mac/Mac Builder',
+ 'win64': 'chromium.win/Win x64 Builder',
+ 'win': 'chromium.win/Win Builder',
+ }
+ assert self._platform in platform_to_directory
+ return platform_to_directory.get(self._platform)
+
+
+def FetchFromCloudStorage(bucket_name, source_path, destination_dir):
+ """Fetches file(s) from the Google Cloud Storage.
+
+ As a side-effect, this prints messages to stdout about what's happening.
+
+ Args:
+ bucket_name: Google Storage bucket name.
+ source_path: Source file path.
+ destination_dir: Destination file path.
+
+ Returns:
+ Local file path of downloaded file if it was downloaded. If the file does
+ not exist in the given bucket, or if there was an error while downloading,
+ None is returned.
+ """
+ target_file = os.path.join(destination_dir, os.path.basename(source_path))
+ gs_url = 'gs://%s/%s' % (bucket_name, source_path)
+ try:
+ if cloud_storage.Exists(bucket_name, source_path):
+ print 'Fetching file from %s...' % gs_url
+ cloud_storage.Get(bucket_name, source_path, target_file)
+ if os.path.exists(target_file):
+ return target_file
+ else:
+ print 'File %s not found in cloud storage.' % gs_url
+ except Exception as e:
+ print 'Exception while fetching from cloud storage: %s' % e
+ if os.path.exists(target_file):
+ os.remove(target_file)
+ return None
+
+
+def Unzip(filename, output_dir, verbose=True):
+ """Extracts a zip archive's contents into the given output directory.
+
+ This was based on ExtractZip from build/scripts/common/chromium_utils.py.
+
+ Args:
+ filename: Name of the zip file to extract.
+ output_dir: Path to the destination directory.
+ verbose: Whether to print out what is being extracted.
+
+ Raises:
+ IOError: The unzip command had a non-zero exit code.
+ RuntimeError: Failed to create the output directory.
+ """
+ _MakeDirectory(output_dir)
+
+ # On Linux and Mac, we use the unzip command because it handles links and
+ # file permissions bits, so achieving this behavior is easier than with
+ # ZipInfo options.
+ #
+ # The Mac Version of unzip unfortunately does not support Zip64, whereas
+ # the python module does, so we have to fall back to the python zip module
+ # on Mac if the file size is greater than 4GB.
+ mac_zip_size_limit = 2 ** 32 # 4GB
+ if (bisect_utils.IsLinuxHost() or
+ (bisect_utils.IsMacHost()
+ and os.path.getsize(filename) < mac_zip_size_limit)):
+ unzip_command = ['unzip', '-o']
+ _UnzipUsingCommand(unzip_command, filename, output_dir)
+ return
+
+ # On Windows, try to use 7z if it is installed, otherwise fall back to the
+ # Python zipfile module. If 7z is not installed, then this may fail if the
+ # zip file is larger than 512MB.
+ sevenzip_path = r'C:\Program Files\7-Zip\7z.exe'
+ if (bisect_utils.IsWindowsHost() and os.path.exists(sevenzip_path)):
+ unzip_command = [sevenzip_path, 'x', '-y']
+ _UnzipUsingCommand(unzip_command, filename, output_dir)
+ return
+
+ _UnzipUsingZipFile(filename, output_dir, verbose)
+
+
+def _UnzipUsingCommand(unzip_command, filename, output_dir):
+ """Extracts a zip file using an external command.
+
+ Args:
+ unzip_command: An unzipping command, as a string list, without the filename.
+ filename: Path to the zip file.
+ output_dir: The directory which the contents should be extracted to.
+
+ Raises:
+ IOError: The command had a non-zero exit code.
+ """
+ absolute_filepath = os.path.abspath(filename)
+ command = unzip_command + [absolute_filepath]
+ return_code = _RunCommandInDirectory(output_dir, command)
+ if return_code:
+ _RemoveDirectoryTree(output_dir)
+ raise IOError('Unzip failed: %s => %s' % (str(command), return_code))
+
+
+def _RunCommandInDirectory(directory, command):
+ """Changes to a directory, runs a command, then changes back."""
+ saved_dir = os.getcwd()
+ os.chdir(directory)
+ return_code = bisect_utils.RunProcess(command)
+ os.chdir(saved_dir)
+ return return_code
+
+
+def _UnzipUsingZipFile(filename, output_dir, verbose=True):
+ """Extracts a zip file using the Python zipfile module."""
+ assert bisect_utils.IsWindowsHost() or bisect_utils.IsMacHost()
+ zf = zipfile.ZipFile(filename)
+ for name in zf.namelist():
+ if verbose:
+ print 'Extracting %s' % name
+ zf.extract(name, output_dir)
+ if bisect_utils.IsMacHost():
+ # Restore file permission bits.
+ mode = zf.getinfo(name).external_attr >> 16
+ os.chmod(os.path.join(output_dir, name), mode)
+
+
+def _MakeDirectory(path):
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+
+def _RemoveDirectoryTree(path):
+ try:
+ if os.path.exists(path):
+ shutil.rmtree(path)
+ except OSError, e:
+ if e.errno != errno.ENOENT:
+ raise
+
+
+def Main(argv):
+ """Downloads and extracts a build based on the command line arguments."""
+ parser = argparse.ArgumentParser()
+ parser.add_argument('builder_type')
+ parser.add_argument('revision')
+ parser.add_argument('output_dir')
+ parser.add_argument('--target-arch', default='ia32')
+ parser.add_argument('--target-platform', default='chromium')
+ parser.add_argument('--deps-patch-sha')
+ args = parser.parse_args(argv[1:])
+
+ FetchBuild(
+ args.builder_type, args.revision, args.output_dir,
+ target_arch=args.target_arch, target_platform=args.target_platform,
+ deps_patch_sha=args.deps_patch_sha)
+
+ print 'Build has been downloaded to and extracted in %s.' % args.output_dir
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(Main(sys.argv))
+
« no previous file with comments | « tools/auto_bisect/bisect_utils.py ('k') | tools/auto_bisect/fetch_build_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698