| Index: tools/bisect-perf-regression.py
|
| diff --git a/tools/bisect-perf-regression.py b/tools/bisect-perf-regression.py
|
| deleted file mode 100755
|
| index 6166f402b3fc0b3eb050632982cd21281f2ca51b..0000000000000000000000000000000000000000
|
| --- a/tools/bisect-perf-regression.py
|
| +++ /dev/null
|
| @@ -1,3406 +0,0 @@
|
| -#!/usr/bin/env python
|
| -# Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
| -# Use of this source code is governed by a BSD-style license that can be
|
| -# found in the LICENSE file.
|
| -
|
| -"""Performance Test Bisect Tool
|
| -
|
| -This script bisects a series of changelists using binary search. It starts at
|
| -a bad revision where a performance metric has regressed, and asks for a last
|
| -known-good revision. It will then binary search across this revision range by
|
| -syncing, building, and running a performance test. If the change is
|
| -suspected to occur as a result of WebKit/V8 changes, the script will
|
| -further bisect changes to those depots and attempt to narrow down the revision
|
| -range.
|
| -
|
| -Example usage using SVN revisions:
|
| -
|
| -./tools/bisect-perf-regression.py -c\
|
| - "out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\
|
| - -g 168222 -b 168232 -m shutdown/simple-user-quit
|
| -
|
| -Be aware that if you're using the git workflow and specify an SVN revision,
|
| -the script will attempt to find the git SHA1 where SVN changes up to that
|
| -revision were merged in.
|
| -
|
| -Example usage using git hashes:
|
| -
|
| -./tools/bisect-perf-regression.py -c\
|
| - "out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\
|
| - -g 1f6e67861535121c5c819c16a666f2436c207e7b\
|
| - -b b732f23b4f81c382db0b23b9035f3dadc7d925bb\
|
| - -m shutdown/simple-user-quit
|
| -"""
|
| -
|
| -import copy
|
| -import datetime
|
| -import errno
|
| -import hashlib
|
| -import math
|
| -import optparse
|
| -import os
|
| -import re
|
| -import shlex
|
| -import shutil
|
| -import StringIO
|
| -import sys
|
| -import time
|
| -import zipfile
|
| -
|
| -sys.path.append(os.path.join(os.path.dirname(__file__), 'telemetry'))
|
| -
|
| -from auto_bisect import bisect_utils
|
| -from auto_bisect import builder
|
| -from auto_bisect import math_utils
|
| -from auto_bisect import request_build
|
| -from auto_bisect import source_control as source_control_module
|
| -from auto_bisect import ttest
|
| -from telemetry.util import cloud_storage
|
| -
|
| -# Below is the map of "depot" names to information about each depot. Each depot
|
| -# is a repository, and in the process of bisecting, revision ranges in these
|
| -# repositories may also be bisected.
|
| -#
|
| -# Each depot information dictionary may contain:
|
| -# src: Path to the working directory.
|
| -# recurse: True if this repository will get bisected.
|
| -# depends: A list of other repositories that are actually part of the same
|
| -# repository in svn. If the repository has any dependent repositories
|
| -# (e.g. skia/src needs skia/include and skia/gyp to be updated), then
|
| -# they are specified here.
|
| -# svn: URL of SVN repository. Needed for git workflow to resolve hashes to
|
| -# SVN revisions.
|
| -# from: Parent depot that must be bisected before this is bisected.
|
| -# deps_var: Key name in vars variable in DEPS file that has revision
|
| -# information.
|
| -DEPOT_DEPS_NAME = {
|
| - 'chromium': {
|
| - 'src': 'src',
|
| - 'recurse': True,
|
| - 'depends': None,
|
| - 'from': ['cros', 'android-chrome'],
|
| - 'viewvc':
|
| - 'http://src.chromium.org/viewvc/chrome?view=revision&revision=',
|
| - 'deps_var': 'chromium_rev'
|
| - },
|
| - 'webkit': {
|
| - 'src': 'src/third_party/WebKit',
|
| - 'recurse': True,
|
| - 'depends': None,
|
| - 'from': ['chromium'],
|
| - 'viewvc':
|
| - 'http://src.chromium.org/viewvc/blink?view=revision&revision=',
|
| - 'deps_var': 'webkit_revision'
|
| - },
|
| - 'angle': {
|
| - 'src': 'src/third_party/angle',
|
| - 'src_old': 'src/third_party/angle_dx11',
|
| - 'recurse': True,
|
| - 'depends': None,
|
| - 'from': ['chromium'],
|
| - 'platform': 'nt',
|
| - 'deps_var': 'angle_revision'
|
| - },
|
| - 'v8': {
|
| - 'src': 'src/v8',
|
| - 'recurse': True,
|
| - 'depends': None,
|
| - 'from': ['chromium'],
|
| - 'custom_deps': bisect_utils.GCLIENT_CUSTOM_DEPS_V8,
|
| - 'viewvc': 'https://code.google.com/p/v8/source/detail?r=',
|
| - 'deps_var': 'v8_revision'
|
| - },
|
| - 'v8_bleeding_edge': {
|
| - 'src': 'src/v8_bleeding_edge',
|
| - 'recurse': True,
|
| - 'depends': None,
|
| - 'svn': 'https://v8.googlecode.com/svn/branches/bleeding_edge',
|
| - 'from': ['v8'],
|
| - 'viewvc': 'https://code.google.com/p/v8/source/detail?r=',
|
| - 'deps_var': 'v8_revision'
|
| - },
|
| - 'skia/src': {
|
| - 'src': 'src/third_party/skia/src',
|
| - 'recurse': True,
|
| - 'svn': 'http://skia.googlecode.com/svn/trunk/src',
|
| - 'depends': ['skia/include', 'skia/gyp'],
|
| - 'from': ['chromium'],
|
| - 'viewvc': 'https://code.google.com/p/skia/source/detail?r=',
|
| - 'deps_var': 'skia_revision'
|
| - },
|
| - 'skia/include': {
|
| - 'src': 'src/third_party/skia/include',
|
| - 'recurse': False,
|
| - 'svn': 'http://skia.googlecode.com/svn/trunk/include',
|
| - 'depends': None,
|
| - 'from': ['chromium'],
|
| - 'viewvc': 'https://code.google.com/p/skia/source/detail?r=',
|
| - 'deps_var': 'None'
|
| - },
|
| - 'skia/gyp': {
|
| - 'src': 'src/third_party/skia/gyp',
|
| - 'recurse': False,
|
| - 'svn': 'http://skia.googlecode.com/svn/trunk/gyp',
|
| - 'depends': None,
|
| - 'from': ['chromium'],
|
| - 'viewvc': 'https://code.google.com/p/skia/source/detail?r=',
|
| - 'deps_var': 'None'
|
| - }
|
| -}
|
| -
|
| -DEPOT_NAMES = DEPOT_DEPS_NAME.keys()
|
| -
|
| -CROS_CHROMEOS_PATTERN = 'chromeos-base/chromeos-chrome'
|
| -
|
| -# Possible return values from BisectPerformanceMetrics.RunTest.
|
| -BUILD_RESULT_SUCCEED = 0
|
| -BUILD_RESULT_FAIL = 1
|
| -BUILD_RESULT_SKIPPED = 2
|
| -
|
| -# Maximum time in seconds to wait after posting build request to the try server.
|
| -# TODO: Change these values based on the actual time taken by buildbots on
|
| -# the try server.
|
| -MAX_MAC_BUILD_TIME = 14400
|
| -MAX_WIN_BUILD_TIME = 14400
|
| -MAX_LINUX_BUILD_TIME = 14400
|
| -
|
| -# The confidence percentage at which confidence can be consider "high".
|
| -HIGH_CONFIDENCE = 95
|
| -
|
| -# Patch template to add a new file, DEPS.sha under src folder.
|
| -# This file contains SHA1 value of the DEPS changes made while bisecting
|
| -# dependency repositories. This patch send along with DEPS patch to try server.
|
| -# When a build requested is posted with a patch, bisect builders on try server,
|
| -# once build is produced, it reads SHA value from this file and appends it
|
| -# to build archive filename.
|
| -DEPS_SHA_PATCH = """diff --git src/DEPS.sha src/DEPS.sha
|
| -new file mode 100644
|
| ---- /dev/null
|
| -+++ src/DEPS.sha
|
| -@@ -0,0 +1 @@
|
| -+%(deps_sha)s
|
| -"""
|
| -
|
| -# The possible values of the --bisect_mode flag, which determines what to
|
| -# use when classifying a revision as "good" or "bad".
|
| -BISECT_MODE_MEAN = 'mean'
|
| -BISECT_MODE_STD_DEV = 'std_dev'
|
| -BISECT_MODE_RETURN_CODE = 'return_code'
|
| -
|
| -# The perf dashboard looks for a string like "Estimated Confidence: 95%"
|
| -# to decide whether or not to cc the author(s). If you change this, please
|
| -# update the perf dashboard as well.
|
| -RESULTS_BANNER = """
|
| -===== BISECT JOB RESULTS =====
|
| -Status: %(status)s
|
| -
|
| -Test Command: %(command)s
|
| -Test Metric: %(metrics)s
|
| -Relative Change: %(change)s
|
| -Estimated Confidence: %(confidence).02f%%"""
|
| -
|
| -# The perf dashboard specifically looks for the string
|
| -# "Author : " to parse out who to cc on a bug. If you change the
|
| -# formatting here, please update the perf dashboard as well.
|
| -RESULTS_REVISION_INFO = """
|
| -===== SUSPECTED CL(s) =====
|
| -Subject : %(subject)s
|
| -Author : %(author)s%(email_info)s%(commit_info)s
|
| -Commit : %(cl)s
|
| -Date : %(cl_date)s"""
|
| -
|
| -REPRO_STEPS_LOCAL = """
|
| -==== INSTRUCTIONS TO REPRODUCE ====
|
| -To run locally:
|
| - - Use the test command given under 'BISECT JOB RESULTS' above.
|
| - - Consider using a profiler. Pass --profiler=list to list available profilers.
|
| -"""
|
| -
|
| -REPRO_STEPS_TRYJOB = """
|
| -To reproduce on a performance try bot:
|
| - 1. Edit run-perf-test.cfg
|
| - 2. Upload your patch with: $ git cl upload --bypass-hooks
|
| - 3. Send to the try server: $ git cl try -m tryserver.chromium.perf -b <bot>
|
| -
|
| -Notes:
|
| - a) Follow the in-file instructions in run-perf-test.cfg.
|
| - b) run-perf-test.cfg is under tools/ or under third_party/WebKit/Tools.
|
| - c) Do your edits preferably under a new git branch.
|
| - d) --browser=release and --browser=android-chromium-testshell are supported
|
| - depending on the platform (desktop|android).
|
| - e) Strip any src/ directories from the head of relative path names.
|
| - f) Make sure to use the appropriate bot on step 3.
|
| -
|
| -For more details please visit
|
| -https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots"""
|
| -
|
| -REPRO_STEPS_TRYJOB_TELEMETRY = """
|
| -To reproduce on a performance try bot:
|
| -%(command)s
|
| -(Where <bot-name> comes from tools/perf/run_benchmark --browser=list)
|
| -
|
| -For more details please visit
|
| -https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots
|
| -"""
|
| -
|
| -RESULTS_THANKYOU = """
|
| -===== THANK YOU FOR CHOOSING BISECT AIRLINES =====
|
| -Visit http://www.chromium.org/developers/core-principles for Chrome's policy
|
| -on perf regressions.
|
| -Contact chrome-perf-dashboard-team with any questions or suggestions about
|
| -bisecting.
|
| -. .-----.
|
| -. .---. \ \==)
|
| -. |PERF\ \ \\
|
| -. | ---------'-------'-----------.
|
| -. . 0 0 0 0 0 0 0 0 0 0 0 0 0 0 |_`-.
|
| -. \_____________.-------._______________)
|
| -. / /
|
| -. / /
|
| -. / /==)
|
| -. ._____."""
|
| -
|
| -
|
| -def _AddAdditionalDepotInfo(depot_info):
|
| - """Adds additional depot info to the global depot variables."""
|
| - global DEPOT_DEPS_NAME
|
| - global DEPOT_NAMES
|
| - DEPOT_DEPS_NAME = dict(DEPOT_DEPS_NAME.items() + depot_info.items())
|
| - DEPOT_NAMES = DEPOT_DEPS_NAME.keys()
|
| -
|
| -
|
| -def ConfidenceScore(good_results_lists, bad_results_lists):
|
| - """Calculates a confidence score.
|
| -
|
| - This score is a percentage which represents our degree of confidence in the
|
| - proposition that the good results and bad results are distinct groups, and
|
| - their differences aren't due to chance alone.
|
| -
|
| -
|
| - Args:
|
| - good_results_lists: A list of lists of "good" result numbers.
|
| - bad_results_lists: A list of lists of "bad" result numbers.
|
| -
|
| - Returns:
|
| - A number in the range [0, 100].
|
| - """
|
| - # If there's only one item in either list, this means only one revision was
|
| - # classified good or bad; this isn't good enough evidence to make a decision.
|
| - # If an empty list was passed, that also implies zero confidence.
|
| - if len(good_results_lists) <= 1 or len(bad_results_lists) <= 1:
|
| - return 0.0
|
| -
|
| - # Flatten the lists of results lists.
|
| - sample1 = sum(good_results_lists, [])
|
| - sample2 = sum(bad_results_lists, [])
|
| -
|
| - # If there were only empty lists in either of the lists (this is unexpected
|
| - # and normally shouldn't happen), then we also want to return 0.
|
| - if not sample1 or not sample2:
|
| - return 0.0
|
| -
|
| - # The p-value is approximately the probability of obtaining the given set
|
| - # of good and bad values just by chance.
|
| - _, _, p_value = ttest.WelchsTTest(sample1, sample2)
|
| - return 100.0 * (1.0 - p_value)
|
| -
|
| -
|
| -def GetSHA1HexDigest(contents):
|
| - """Returns SHA1 hex digest of the given string."""
|
| - return hashlib.sha1(contents).hexdigest()
|
| -
|
| -
|
| -def GetZipFileName(build_revision=None, target_arch='ia32', patch_sha=None):
|
| - """Gets the archive file name for the given revision."""
|
| - def PlatformName():
|
| - """Return a string to be used in paths for the platform."""
|
| - if bisect_utils.IsWindowsHost():
|
| - # Build archive for x64 is still stored with the "win32" suffix.
|
| - # See chromium_utils.PlatformName().
|
| - if bisect_utils.Is64BitWindows() and target_arch == 'x64':
|
| - return 'win32'
|
| - return 'win32'
|
| - if bisect_utils.IsLinuxHost():
|
| - # Android builds are also archived with the "full-build-linux prefix.
|
| - return 'linux'
|
| - if bisect_utils.IsMacHost():
|
| - return 'mac'
|
| - raise NotImplementedError('Unknown platform "%s".' % sys.platform)
|
| -
|
| - base_name = 'full-build-%s' % PlatformName()
|
| - if not build_revision:
|
| - return base_name
|
| - if patch_sha:
|
| - build_revision = '%s_%s' % (build_revision , patch_sha)
|
| - return '%s_%s.zip' % (base_name, build_revision)
|
| -
|
| -
|
| -def GetRemoteBuildPath(build_revision, target_platform='chromium',
|
| - target_arch='ia32', patch_sha=None):
|
| - """Returns the URL to download the build from."""
|
| - def GetGSRootFolderName(target_platform):
|
| - """Returns the Google Cloud Storage root folder name."""
|
| - if bisect_utils.IsWindowsHost():
|
| - if bisect_utils.Is64BitWindows() and target_arch == 'x64':
|
| - return 'Win x64 Builder'
|
| - return 'Win Builder'
|
| - if bisect_utils.IsLinuxHost():
|
| - if target_platform == 'android':
|
| - return 'android_perf_rel'
|
| - return 'Linux Builder'
|
| - if bisect_utils.IsMacHost():
|
| - return 'Mac Builder'
|
| - raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
|
| -
|
| - base_filename = GetZipFileName(
|
| - build_revision, target_arch, patch_sha)
|
| - builder_folder = GetGSRootFolderName(target_platform)
|
| - return '%s/%s' % (builder_folder, base_filename)
|
| -
|
| -
|
| -def FetchFromCloudStorage(bucket_name, source_path, destination_path):
|
| - """Fetches file(s) from the Google Cloud Storage.
|
| -
|
| - Args:
|
| - bucket_name: Google Storage bucket name.
|
| - source_path: Source file path.
|
| - destination_path: Destination file path.
|
| -
|
| - Returns:
|
| - Downloaded file path if exists, otherwise None.
|
| - """
|
| - target_file = os.path.join(destination_path, os.path.basename(source_path))
|
| - try:
|
| - if cloud_storage.Exists(bucket_name, source_path):
|
| - print 'Fetching file from gs//%s/%s ...' % (bucket_name, source_path)
|
| - cloud_storage.Get(bucket_name, source_path, destination_path)
|
| - if os.path.exists(target_file):
|
| - return target_file
|
| - else:
|
| - print ('File gs://%s/%s not found in cloud storage.' % (
|
| - bucket_name, source_path))
|
| - except Exception as e:
|
| - print 'Something went wrong while fetching file from cloud: %s' % e
|
| - if os.path.exists(target_file):
|
| - os.remove(target_file)
|
| - return None
|
| -
|
| -
|
| -# This is copied from build/scripts/common/chromium_utils.py.
|
| -def MaybeMakeDirectory(*path):
|
| - """Creates an entire path, if it doesn't already exist."""
|
| - file_path = os.path.join(*path)
|
| - try:
|
| - os.makedirs(file_path)
|
| - except OSError as e:
|
| - if e.errno != errno.EEXIST:
|
| - return False
|
| - return True
|
| -
|
| -
|
| -# This was copied from build/scripts/common/chromium_utils.py.
|
| -def ExtractZip(filename, output_dir, verbose=True):
|
| - """ Extract the zip archive in the output directory."""
|
| - MaybeMakeDirectory(output_dir)
|
| -
|
| - # On Linux and Mac, we use the unzip command as it will
|
| - # handle links and file bits (executable), which is much
|
| - # easier then trying to do that with ZipInfo options.
|
| - #
|
| - # The Mac Version of unzip unfortunately does not support Zip64, whereas
|
| - # the python module does, so we have to fall back to the python zip module
|
| - # on Mac if the file size is greater than 4GB.
|
| - #
|
| - # On Windows, try to use 7z if it is installed, otherwise fall back to python
|
| - # zip module and pray we don't have files larger than 512MB to unzip.
|
| - unzip_cmd = None
|
| - if ((bisect_utils.IsMacHost()
|
| - and os.path.getsize(filename) < 4 * 1024 * 1024 * 1024)
|
| - or bisect_utils.IsLinuxHost()):
|
| - unzip_cmd = ['unzip', '-o']
|
| - elif (bisect_utils.IsWindowsHost()
|
| - and os.path.exists('C:\\Program Files\\7-Zip\\7z.exe')):
|
| - unzip_cmd = ['C:\\Program Files\\7-Zip\\7z.exe', 'x', '-y']
|
| -
|
| - if unzip_cmd:
|
| - # Make sure path is absolute before changing directories.
|
| - filepath = os.path.abspath(filename)
|
| - saved_dir = os.getcwd()
|
| - os.chdir(output_dir)
|
| - command = unzip_cmd + [filepath]
|
| - result = bisect_utils.RunProcess(command)
|
| - os.chdir(saved_dir)
|
| - if result:
|
| - raise IOError('unzip failed: %s => %s' % (str(command), result))
|
| - else:
|
| - assert bisect_utils.IsWindowsHost() or bisect_utils.IsMacHost()
|
| - zf = zipfile.ZipFile(filename)
|
| - for name in zf.namelist():
|
| - if verbose:
|
| - print 'Extracting %s' % name
|
| - zf.extract(name, output_dir)
|
| - if bisect_utils.IsMacHost():
|
| - # Restore permission bits.
|
| - os.chmod(os.path.join(output_dir, name),
|
| - zf.getinfo(name).external_attr >> 16L)
|
| -
|
| -
|
| -def WriteStringToFile(text, file_name):
|
| - """Writes text to a file, raising an RuntimeError on failure."""
|
| - try:
|
| - with open(file_name, 'wb') as f:
|
| - f.write(text)
|
| - except IOError:
|
| - raise RuntimeError('Error writing to file [%s]' % file_name )
|
| -
|
| -
|
| -def ReadStringFromFile(file_name):
|
| - """Writes text to a file, raising an RuntimeError on failure."""
|
| - try:
|
| - with open(file_name) as f:
|
| - return f.read()
|
| - except IOError:
|
| - raise RuntimeError('Error reading file [%s]' % file_name )
|
| -
|
| -
|
| -def ChangeBackslashToSlashInPatch(diff_text):
|
| - """Formats file paths in the given patch text to Unix-style paths."""
|
| - if not diff_text:
|
| - return None
|
| - diff_lines = diff_text.split('\n')
|
| - for i in range(len(diff_lines)):
|
| - line = diff_lines[i]
|
| - if line.startswith('--- ') or line.startswith('+++ '):
|
| - diff_lines[i] = line.replace('\\', '/')
|
| - return '\n'.join(diff_lines)
|
| -
|
| -
|
| -def _ParseRevisionsFromDEPSFileManually(deps_file_contents):
|
| - """Parses the vars section of the DEPS file using regular expressions.
|
| -
|
| - Args:
|
| - deps_file_contents: The DEPS file contents as a string.
|
| -
|
| - Returns:
|
| - A dictionary in the format {depot: revision} if successful, otherwise None.
|
| - """
|
| - # We'll parse the "vars" section of the DEPS file.
|
| - rxp = re.compile('vars = {(?P<vars_body>[^}]+)', re.MULTILINE)
|
| - re_results = rxp.search(deps_file_contents)
|
| -
|
| - if not re_results:
|
| - return None
|
| -
|
| - # We should be left with a series of entries in the vars component of
|
| - # the DEPS file with the following format:
|
| - # 'depot_name': 'revision',
|
| - vars_body = re_results.group('vars_body')
|
| - rxp = re.compile("'(?P<depot_body>[\w_-]+)':[\s]+'(?P<rev_body>[\w@]+)'",
|
| - re.MULTILINE)
|
| - re_results = rxp.findall(vars_body)
|
| -
|
| - return dict(re_results)
|
| -
|
| -
|
| -def _WaitUntilBuildIsReady(
|
| - fetch_build, bot_name, builder_host, builder_port, build_request_id,
|
| - max_timeout):
|
| - """Waits until build is produced by bisect builder on try server.
|
| -
|
| - Args:
|
| - fetch_build: Function to check and download build from cloud storage.
|
| - bot_name: Builder bot name on try server.
|
| - builder_host Try server host name.
|
| - builder_port: Try server port.
|
| - build_request_id: A unique ID of the build request posted to try server.
|
| - max_timeout: Maximum time to wait for the build.
|
| -
|
| - Returns:
|
| - Downloaded archive file path if exists, otherwise None.
|
| - """
|
| - # Build number on the try server.
|
| - build_num = None
|
| - # Interval to check build on cloud storage.
|
| - poll_interval = 60
|
| - # Interval to check build status on try server in seconds.
|
| - status_check_interval = 600
|
| - last_status_check = time.time()
|
| - start_time = time.time()
|
| - while True:
|
| - # Checks for build on gs://chrome-perf and download if exists.
|
| - res = fetch_build()
|
| - if res:
|
| - return (res, 'Build successfully found')
|
| - elapsed_status_check = time.time() - last_status_check
|
| - # To avoid overloading try server with status check requests, we check
|
| - # build status for every 10 minutes.
|
| - if elapsed_status_check > status_check_interval:
|
| - last_status_check = time.time()
|
| - if not build_num:
|
| - # Get the build number on try server for the current build.
|
| - build_num = request_build.GetBuildNumFromBuilder(
|
| - build_request_id, bot_name, builder_host, builder_port)
|
| - # Check the status of build using the build number.
|
| - # Note: Build is treated as PENDING if build number is not found
|
| - # on the the try server.
|
| - build_status, status_link = request_build.GetBuildStatus(
|
| - build_num, bot_name, builder_host, builder_port)
|
| - if build_status == request_build.FAILED:
|
| - return (None, 'Failed to produce build, log: %s' % status_link)
|
| - elapsed_time = time.time() - start_time
|
| - if elapsed_time > max_timeout:
|
| - return (None, 'Timed out: %ss without build' % max_timeout)
|
| -
|
| - print 'Time elapsed: %ss without build.' % elapsed_time
|
| - time.sleep(poll_interval)
|
| - # For some reason, mac bisect bots were not flushing stdout periodically.
|
| - # As a result buildbot command is timed-out. Flush stdout on all platforms
|
| - # while waiting for build.
|
| - sys.stdout.flush()
|
| -
|
| -
|
| -def _UpdateV8Branch(deps_content):
|
| - """Updates V8 branch in DEPS file to process v8_bleeding_edge.
|
| -
|
| - Check for "v8_branch" in DEPS file if exists update its value
|
| - with v8_bleeding_edge branch. Note: "v8_branch" is added to DEPS
|
| - variable from DEPS revision 254916, therefore check for "src/v8":
|
| - <v8 source path> in DEPS in order to support prior DEPS revisions
|
| - and update it.
|
| -
|
| - Args:
|
| - deps_content: DEPS file contents to be modified.
|
| -
|
| - Returns:
|
| - Modified DEPS file contents as a string.
|
| - """
|
| - new_branch = r'branches/bleeding_edge'
|
| - v8_branch_pattern = re.compile(r'(?<="v8_branch": ")(.*)(?=")')
|
| - if re.search(v8_branch_pattern, deps_content):
|
| - deps_content = re.sub(v8_branch_pattern, new_branch, deps_content)
|
| - else:
|
| - # Replaces the branch assigned to "src/v8" key in DEPS file.
|
| - # Format of "src/v8" in DEPS:
|
| - # "src/v8":
|
| - # (Var("googlecode_url") % "v8") + "/trunk@" + Var("v8_revision"),
|
| - # So, "/trunk@" is replace with "/branches/bleeding_edge@"
|
| - v8_src_pattern = re.compile(
|
| - r'(?<="v8"\) \+ "/)(.*)(?=@" \+ Var\("v8_revision"\))', re.MULTILINE)
|
| - if re.search(v8_src_pattern, deps_content):
|
| - deps_content = re.sub(v8_src_pattern, new_branch, deps_content)
|
| - return deps_content
|
| -
|
| -
|
| -def _UpdateDEPSForAngle(revision, depot, deps_file):
|
| - """Updates DEPS file with new revision for Angle repository.
|
| -
|
| - This is a hack for Angle depot case because, in DEPS file "vars" dictionary
|
| - variable contains "angle_revision" key that holds git hash instead of
|
| - SVN revision.
|
| -
|
| - And sometimes "angle_revision" key is not specified in "vars" variable,
|
| - in such cases check "deps" dictionary variable that matches
|
| - angle.git@[a-fA-F0-9]{40}$ and replace git hash.
|
| - """
|
| - deps_var = DEPOT_DEPS_NAME[depot]['deps_var']
|
| - try:
|
| - deps_contents = ReadStringFromFile(deps_file)
|
| - # Check whether the depot and revision pattern in DEPS file vars variable
|
| - # e.g. "angle_revision": "fa63e947cb3eccf463648d21a05d5002c9b8adfa".
|
| - angle_rev_pattern = re.compile(r'(?<="%s": ")([a-fA-F0-9]{40})(?=")' %
|
| - deps_var, re.MULTILINE)
|
| - match = re.search(angle_rev_pattern % deps_var, deps_contents)
|
| - if match:
|
| - # Update the revision information for the given depot
|
| - new_data = re.sub(angle_rev_pattern, revision, deps_contents)
|
| - else:
|
| - # Check whether the depot and revision pattern in DEPS file deps
|
| - # variable. e.g.,
|
| - # "src/third_party/angle": Var("chromium_git") +
|
| - # "/angle/angle.git@fa63e947cb3eccf463648d21a05d5002c9b8adfa",.
|
| - angle_rev_pattern = re.compile(
|
| - r'(?<=angle\.git@)([a-fA-F0-9]{40})(?=")', re.MULTILINE)
|
| - match = re.search(angle_rev_pattern, deps_contents)
|
| - if not match:
|
| - print 'Could not find angle revision information in DEPS file.'
|
| - return False
|
| - new_data = re.sub(angle_rev_pattern, revision, deps_contents)
|
| - # Write changes to DEPS file
|
| - WriteStringToFile(new_data, deps_file)
|
| - return True
|
| - except IOError, e:
|
| - print 'Something went wrong while updating DEPS file, %s' % e
|
| - return False
|
| -
|
| -
|
| -def _TryParseHistogramValuesFromOutput(metric, text):
|
| - """Attempts to parse a metric in the format HISTOGRAM <graph: <trace>.
|
| -
|
| - Args:
|
| - metric: The metric as a list of [<trace>, <value>] strings.
|
| - text: The text to parse the metric values from.
|
| -
|
| - Returns:
|
| - A list of floating point numbers found, [] if none were found.
|
| - """
|
| - metric_formatted = 'HISTOGRAM %s: %s= ' % (metric[0], metric[1])
|
| -
|
| - text_lines = text.split('\n')
|
| - values_list = []
|
| -
|
| - for current_line in text_lines:
|
| - if metric_formatted in current_line:
|
| - current_line = current_line[len(metric_formatted):]
|
| -
|
| - try:
|
| - histogram_values = eval(current_line)
|
| -
|
| - for b in histogram_values['buckets']:
|
| - average_for_bucket = float(b['high'] + b['low']) * 0.5
|
| - # Extends the list with N-elements with the average for that bucket.
|
| - values_list.extend([average_for_bucket] * b['count'])
|
| - except Exception:
|
| - pass
|
| -
|
| - return values_list
|
| -
|
| -
|
| -def _TryParseResultValuesFromOutput(metric, text):
|
| - """Attempts to parse a metric in the format RESULT <graph>: <trace>= ...
|
| -
|
| - Args:
|
| - metric: The metric as a list of [<trace>, <value>] string pairs.
|
| - text: The text to parse the metric values from.
|
| -
|
| - Returns:
|
| - A list of floating point numbers found.
|
| - """
|
| - # Format is: RESULT <graph>: <trace>= <value> <units>
|
| - metric_re = re.escape('RESULT %s: %s=' % (metric[0], metric[1]))
|
| -
|
| - # The log will be parsed looking for format:
|
| - # <*>RESULT <graph_name>: <trace_name>= <value>
|
| - single_result_re = re.compile(
|
| - metric_re + '\s*(?P<VALUE>[-]?\d*(\.\d*)?)')
|
| -
|
| - # The log will be parsed looking for format:
|
| - # <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...]
|
| - multi_results_re = re.compile(
|
| - metric_re + '\s*\[\s*(?P<VALUES>[-]?[\d\., ]+)\s*\]')
|
| -
|
| - # The log will be parsed looking for format:
|
| - # <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>}
|
| - mean_stddev_re = re.compile(
|
| - metric_re +
|
| - '\s*\{\s*(?P<MEAN>[-]?\d*(\.\d*)?),\s*(?P<STDDEV>\d+(\.\d*)?)\s*\}')
|
| -
|
| - text_lines = text.split('\n')
|
| - values_list = []
|
| - for current_line in text_lines:
|
| - # Parse the output from the performance test for the metric we're
|
| - # interested in.
|
| - single_result_match = single_result_re.search(current_line)
|
| - multi_results_match = multi_results_re.search(current_line)
|
| - mean_stddev_match = mean_stddev_re.search(current_line)
|
| - if (not single_result_match is None and
|
| - single_result_match.group('VALUE')):
|
| - values_list += [single_result_match.group('VALUE')]
|
| - elif (not multi_results_match is None and
|
| - multi_results_match.group('VALUES')):
|
| - metric_values = multi_results_match.group('VALUES')
|
| - values_list += metric_values.split(',')
|
| - elif (not mean_stddev_match is None and
|
| - mean_stddev_match.group('MEAN')):
|
| - values_list += [mean_stddev_match.group('MEAN')]
|
| -
|
| - values_list = [float(v) for v in values_list
|
| - if bisect_utils.IsStringFloat(v)]
|
| -
|
| - # If the metric is times/t, we need to sum the timings in order to get
|
| - # similar regression results as the try-bots.
|
| - metrics_to_sum = [
|
| - ['times', 't'],
|
| - ['times', 'page_load_time'],
|
| - ['cold_times', 'page_load_time'],
|
| - ['warm_times', 'page_load_time'],
|
| - ]
|
| -
|
| - if metric in metrics_to_sum:
|
| - if values_list:
|
| - values_list = [reduce(lambda x, y: float(x) + float(y), values_list)]
|
| -
|
| - return values_list
|
| -
|
| -
|
| -def _ParseMetricValuesFromOutput(metric, text):
|
| - """Parses output from performance_ui_tests and retrieves the results for
|
| - a given metric.
|
| -
|
| - Args:
|
| - metric: The metric as a list of [<trace>, <value>] strings.
|
| - text: The text to parse the metric values from.
|
| -
|
| - Returns:
|
| - A list of floating point numbers found.
|
| - """
|
| - metric_values = _TryParseResultValuesFromOutput(metric, text)
|
| -
|
| - if not metric_values:
|
| - metric_values = _TryParseHistogramValuesFromOutput(metric, text)
|
| -
|
| - return metric_values
|
| -
|
| -
|
| -def _GenerateProfileIfNecessary(command_args):
|
| - """Checks the command line of the performance test for dependencies on
|
| - profile generation, and runs tools/perf/generate_profile as necessary.
|
| -
|
| - Args:
|
| - command_args: Command line being passed to performance test, as a list.
|
| -
|
| - Returns:
|
| - False if profile generation was necessary and failed, otherwise True.
|
| - """
|
| - if '--profile-dir' in ' '.join(command_args):
|
| - # If we were using python 2.7+, we could just use the argparse
|
| - # module's parse_known_args to grab --profile-dir. Since some of the
|
| - # bots still run 2.6, have to grab the arguments manually.
|
| - arg_dict = {}
|
| - args_to_parse = ['--profile-dir', '--browser']
|
| -
|
| - for arg_to_parse in args_to_parse:
|
| - for i, current_arg in enumerate(command_args):
|
| - if arg_to_parse in current_arg:
|
| - current_arg_split = current_arg.split('=')
|
| -
|
| - # Check 2 cases, --arg=<val> and --arg <val>
|
| - if len(current_arg_split) == 2:
|
| - arg_dict[arg_to_parse] = current_arg_split[1]
|
| - elif i + 1 < len(command_args):
|
| - arg_dict[arg_to_parse] = command_args[i+1]
|
| -
|
| - path_to_generate = os.path.join('tools', 'perf', 'generate_profile')
|
| -
|
| - if arg_dict.has_key('--profile-dir') and arg_dict.has_key('--browser'):
|
| - profile_path, profile_type = os.path.split(arg_dict['--profile-dir'])
|
| - return not bisect_utils.RunProcess(['python', path_to_generate,
|
| - '--profile-type-to-generate', profile_type,
|
| - '--browser', arg_dict['--browser'], '--output-dir', profile_path])
|
| - return False
|
| - return True
|
| -
|
| -
|
| -def _AddRevisionsIntoRevisionData(revisions, depot, sort, revision_data):
|
| - """Adds new revisions to the revision_data dictionary and initializes them.
|
| -
|
| - Args:
|
| - revisions: List of revisions to add.
|
| - depot: Depot that's currently in use (src, webkit, etc...)
|
| - sort: Sorting key for displaying revisions.
|
| - revision_data: A dictionary to add the new revisions into.
|
| - Existing revisions will have their sort keys adjusted.
|
| - """
|
| - num_depot_revisions = len(revisions)
|
| -
|
| - for _, v in revision_data.iteritems():
|
| - if v['sort'] > sort:
|
| - v['sort'] += num_depot_revisions
|
| -
|
| - for i in xrange(num_depot_revisions):
|
| - r = revisions[i]
|
| - revision_data[r] = {
|
| - 'revision' : r,
|
| - 'depot' : depot,
|
| - 'value' : None,
|
| - 'perf_time' : 0,
|
| - 'build_time' : 0,
|
| - 'passed' : '?',
|
| - 'sort' : i + sort + 1,
|
| - }
|
| -
|
| -
|
| -def _PrintThankYou():
|
| - print RESULTS_THANKYOU
|
| -
|
| -
|
| -def _PrintTableRow(column_widths, row_data):
|
| - """Prints out a row in a formatted table that has columns aligned.
|
| -
|
| - Args:
|
| - column_widths: A list of column width numbers.
|
| - row_data: A list of items for each column in this row.
|
| - """
|
| - assert len(column_widths) == len(row_data)
|
| - text = ''
|
| - for i in xrange(len(column_widths)):
|
| - current_row_data = row_data[i].center(column_widths[i], ' ')
|
| - text += ('%%%ds' % column_widths[i]) % current_row_data
|
| - print text
|
| -
|
| -
|
| -def _PrintStepTime(revision_data_sorted):
|
| - """Prints information about how long various steps took.
|
| -
|
| - Args:
|
| - revision_data_sorted: The sorted list of revision data dictionaries."""
|
| - step_perf_time_avg = 0.0
|
| - step_build_time_avg = 0.0
|
| - step_count = 0.0
|
| - for _, current_data in revision_data_sorted:
|
| - if current_data['value']:
|
| - step_perf_time_avg += current_data['perf_time']
|
| - step_build_time_avg += current_data['build_time']
|
| - step_count += 1
|
| - if step_count:
|
| - step_perf_time_avg = step_perf_time_avg / step_count
|
| - step_build_time_avg = step_build_time_avg / step_count
|
| - print
|
| - print 'Average build time : %s' % datetime.timedelta(
|
| - seconds=int(step_build_time_avg))
|
| - print 'Average test time : %s' % datetime.timedelta(
|
| - seconds=int(step_perf_time_avg))
|
| -
|
| -
|
| -def _FindOtherRegressions(revision_data_sorted, bad_greater_than_good):
|
| - """Compiles a list of other possible regressions from the revision data.
|
| -
|
| - Args:
|
| - revision_data_sorted: Sorted list of (revision, revision data) pairs.
|
| - bad_greater_than_good: Whether the result value at the "bad" revision is
|
| - numerically greater than the result value at the "good" revision.
|
| -
|
| - Returns:
|
| - A list of [current_rev, previous_rev, confidence] for other places where
|
| - there may have been a regression.
|
| - """
|
| - other_regressions = []
|
| - previous_values = []
|
| - previous_id = None
|
| - for current_id, current_data in revision_data_sorted:
|
| - current_values = current_data['value']
|
| - if current_values:
|
| - current_values = current_values['values']
|
| - if previous_values:
|
| - confidence = ConfidenceScore(previous_values, [current_values])
|
| - mean_of_prev_runs = math_utils.Mean(sum(previous_values, []))
|
| - mean_of_current_runs = math_utils.Mean(current_values)
|
| -
|
| - # Check that the potential regression is in the same direction as
|
| - # the overall regression. If the mean of the previous runs < the
|
| - # mean of the current runs, this local regression is in same
|
| - # direction.
|
| - prev_less_than_current = mean_of_prev_runs < mean_of_current_runs
|
| - is_same_direction = (prev_less_than_current if
|
| - bad_greater_than_good else not prev_less_than_current)
|
| -
|
| - # Only report potential regressions with high confidence.
|
| - if is_same_direction and confidence > 50:
|
| - other_regressions.append([current_id, previous_id, confidence])
|
| - previous_values.append(current_values)
|
| - previous_id = current_id
|
| - return other_regressions
|
| -
|
| -
|
| -class BisectPerformanceMetrics(object):
|
| - """This class contains functionality to perform a bisection of a range of
|
| - revisions to narrow down where performance regressions may have occurred.
|
| -
|
| - The main entry-point is the Run method.
|
| - """
|
| -
|
| - def __init__(self, source_control, opts):
|
| - super(BisectPerformanceMetrics, self).__init__()
|
| -
|
| - self.opts = opts
|
| - self.source_control = source_control
|
| - self.src_cwd = os.getcwd()
|
| - self.cros_cwd = os.path.join(os.getcwd(), '..', 'cros')
|
| - self.depot_cwd = {}
|
| - self.cleanup_commands = []
|
| - self.warnings = []
|
| - self.builder = builder.Builder.FromOpts(opts)
|
| -
|
| - for d in DEPOT_NAMES:
|
| - # The working directory of each depot is just the path to the depot, but
|
| - # since we're already in 'src', we can skip that part.
|
| -
|
| - self.depot_cwd[d] = os.path.join(
|
| - self.src_cwd, DEPOT_DEPS_NAME[d]['src'][4:])
|
| -
|
| - def PerformCleanup(self):
|
| - """Performs cleanup when script is finished."""
|
| - os.chdir(self.src_cwd)
|
| - for c in self.cleanup_commands:
|
| - if c[0] == 'mv':
|
| - shutil.move(c[1], c[2])
|
| - else:
|
| - assert False, 'Invalid cleanup command.'
|
| -
|
| - def GetRevisionList(self, depot, bad_revision, good_revision):
|
| - """Retrieves a list of all the commits between the bad revision and
|
| - last known good revision."""
|
| -
|
| - revision_work_list = []
|
| -
|
| - if depot == 'cros':
|
| - revision_range_start = good_revision
|
| - revision_range_end = bad_revision
|
| -
|
| - cwd = os.getcwd()
|
| - self.ChangeToDepotWorkingDirectory('cros')
|
| -
|
| - # Print the commit timestamps for every commit in the revision time
|
| - # range. We'll sort them and bisect by that. There is a remote chance that
|
| - # 2 (or more) commits will share the exact same timestamp, but it's
|
| - # probably safe to ignore that case.
|
| - cmd = ['repo', 'forall', '-c',
|
| - 'git log --format=%%ct --before=%d --after=%d' % (
|
| - revision_range_end, revision_range_start)]
|
| - output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd)
|
| -
|
| - assert not return_code, ('An error occurred while running '
|
| - '"%s"' % ' '.join(cmd))
|
| -
|
| - os.chdir(cwd)
|
| -
|
| - revision_work_list = list(set(
|
| - [int(o) for o in output.split('\n') if bisect_utils.IsStringInt(o)]))
|
| - revision_work_list = sorted(revision_work_list, reverse=True)
|
| - else:
|
| - cwd = self._GetDepotDirectory(depot)
|
| - revision_work_list = self.source_control.GetRevisionList(bad_revision,
|
| - good_revision, cwd=cwd)
|
| -
|
| - return revision_work_list
|
| -
|
| - def _GetV8BleedingEdgeFromV8TrunkIfMappable(self, revision):
|
| - commit_position = self.source_control.GetCommitPosition(revision)
|
| -
|
| - if bisect_utils.IsStringInt(commit_position):
|
| - # V8 is tricky to bisect, in that there are only a few instances when
|
| - # we can dive into bleeding_edge and get back a meaningful result.
|
| - # Try to detect a V8 "business as usual" case, which is when:
|
| - # 1. trunk revision N has description "Version X.Y.Z"
|
| - # 2. bleeding_edge revision (N-1) has description "Prepare push to
|
| - # trunk. Now working on X.Y.(Z+1)."
|
| - #
|
| - # As of 01/24/2014, V8 trunk descriptions are formatted:
|
| - # "Version 3.X.Y (based on bleeding_edge revision rZ)"
|
| - # So we can just try parsing that out first and fall back to the old way.
|
| - v8_dir = self._GetDepotDirectory('v8')
|
| - v8_bleeding_edge_dir = self._GetDepotDirectory('v8_bleeding_edge')
|
| -
|
| - revision_info = self.source_control.QueryRevisionInfo(revision,
|
| - cwd=v8_dir)
|
| -
|
| - version_re = re.compile("Version (?P<values>[0-9,.]+)")
|
| -
|
| - regex_results = version_re.search(revision_info['subject'])
|
| -
|
| - if regex_results:
|
| - git_revision = None
|
| -
|
| - # Look for "based on bleeding_edge" and parse out revision
|
| - if 'based on bleeding_edge' in revision_info['subject']:
|
| - try:
|
| - bleeding_edge_revision = revision_info['subject'].split(
|
| - 'bleeding_edge revision r')[1]
|
| - bleeding_edge_revision = int(bleeding_edge_revision.split(')')[0])
|
| - git_revision = self.source_control.ResolveToRevision(
|
| - bleeding_edge_revision, 'v8_bleeding_edge', DEPOT_DEPS_NAME, 1,
|
| - cwd=v8_bleeding_edge_dir)
|
| - return git_revision
|
| - except (IndexError, ValueError):
|
| - pass
|
| -
|
| - if not git_revision:
|
| - # Wasn't successful, try the old way of looking for "Prepare push to"
|
| - git_revision = self.source_control.ResolveToRevision(
|
| - int(commit_position) - 1, 'v8_bleeding_edge', DEPOT_DEPS_NAME, -1,
|
| - cwd=v8_bleeding_edge_dir)
|
| -
|
| - if git_revision:
|
| - revision_info = self.source_control.QueryRevisionInfo(git_revision,
|
| - cwd=v8_bleeding_edge_dir)
|
| -
|
| - if 'Prepare push to trunk' in revision_info['subject']:
|
| - return git_revision
|
| - return None
|
| -
|
| - def _GetNearestV8BleedingEdgeFromTrunk(self, revision, search_forward=True):
|
| - cwd = self._GetDepotDirectory('v8')
|
| - cmd = ['log', '--format=%ct', '-1', revision]
|
| - output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
|
| - commit_time = int(output)
|
| - commits = []
|
| -
|
| - if search_forward:
|
| - cmd = ['log', '--format=%H', '-10', '--after=%d' % commit_time,
|
| - 'origin/master']
|
| - output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
|
| - output = output.split()
|
| - commits = output
|
| - commits = reversed(commits)
|
| - else:
|
| - cmd = ['log', '--format=%H', '-10', '--before=%d' % commit_time,
|
| - 'origin/master']
|
| - output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
|
| - output = output.split()
|
| - commits = output
|
| -
|
| - bleeding_edge_revision = None
|
| -
|
| - for c in commits:
|
| - bleeding_edge_revision = self._GetV8BleedingEdgeFromV8TrunkIfMappable(c)
|
| - if bleeding_edge_revision:
|
| - break
|
| -
|
| - return bleeding_edge_revision
|
| -
|
| - def _ParseRevisionsFromDEPSFile(self, depot):
|
| - """Parses the local DEPS file to determine blink/skia/v8 revisions which may
|
| - be needed if the bisect recurses into those depots later.
|
| -
|
| - Args:
|
| - depot: Name of depot being bisected.
|
| -
|
| - Returns:
|
| - A dict in the format {depot:revision} if successful, otherwise None.
|
| - """
|
| - try:
|
| - deps_data = {
|
| - 'Var': lambda _: deps_data["vars"][_],
|
| - 'From': lambda *args: None,
|
| - }
|
| -
|
| - deps_file = bisect_utils.FILE_DEPS_GIT
|
| - if not os.path.exists(deps_file):
|
| - deps_file = bisect_utils.FILE_DEPS
|
| - execfile(deps_file, {}, deps_data)
|
| - deps_data = deps_data['deps']
|
| -
|
| - rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)")
|
| - results = {}
|
| - for depot_name, depot_data in DEPOT_DEPS_NAME.iteritems():
|
| - if (depot_data.get('platform') and
|
| - depot_data.get('platform') != os.name):
|
| - continue
|
| -
|
| - if (depot_data.get('recurse') and depot in depot_data.get('from')):
|
| - depot_data_src = depot_data.get('src') or depot_data.get('src_old')
|
| - src_dir = deps_data.get(depot_data_src)
|
| - if src_dir:
|
| - self.depot_cwd[depot_name] = os.path.join(self.src_cwd,
|
| - depot_data_src[4:])
|
| - re_results = rxp.search(src_dir)
|
| - if re_results:
|
| - results[depot_name] = re_results.group('revision')
|
| - else:
|
| - warning_text = ('Could not parse revision for %s while bisecting '
|
| - '%s' % (depot_name, depot))
|
| - if not warning_text in self.warnings:
|
| - self.warnings.append(warning_text)
|
| - else:
|
| - results[depot_name] = None
|
| - return results
|
| - except ImportError:
|
| - deps_file_contents = ReadStringFromFile(deps_file)
|
| - parse_results = _ParseRevisionsFromDEPSFileManually(deps_file_contents)
|
| - results = {}
|
| - for depot_name, depot_revision in parse_results.iteritems():
|
| - depot_revision = depot_revision.strip('@')
|
| - print depot_name, depot_revision
|
| - for current_name, current_data in DEPOT_DEPS_NAME.iteritems():
|
| - if (current_data.has_key('deps_var') and
|
| - current_data['deps_var'] == depot_name):
|
| - src_name = current_name
|
| - results[src_name] = depot_revision
|
| - break
|
| - return results
|
| -
|
| - def _Get3rdPartyRevisions(self, depot):
|
| - """Parses the DEPS file to determine WebKit/v8/etc... versions.
|
| -
|
| - Args:
|
| - depot: A depot name. Should be in the DEPOT_NAMES list.
|
| -
|
| - Returns:
|
| - A dict in the format {depot: revision} if successful, otherwise None.
|
| - """
|
| - cwd = os.getcwd()
|
| - self.ChangeToDepotWorkingDirectory(depot)
|
| -
|
| - results = {}
|
| -
|
| - if depot == 'chromium' or depot == 'android-chrome':
|
| - results = self._ParseRevisionsFromDEPSFile(depot)
|
| - os.chdir(cwd)
|
| -
|
| - if depot == 'cros':
|
| - cmd = [
|
| - bisect_utils.CROS_SDK_PATH,
|
| - '--',
|
| - 'portageq-%s' % self.opts.cros_board,
|
| - 'best_visible',
|
| - '/build/%s' % self.opts.cros_board,
|
| - 'ebuild',
|
| - CROS_CHROMEOS_PATTERN
|
| - ]
|
| - output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd)
|
| -
|
| - assert not return_code, ('An error occurred while running '
|
| - '"%s"' % ' '.join(cmd))
|
| -
|
| - if len(output) > CROS_CHROMEOS_PATTERN:
|
| - output = output[len(CROS_CHROMEOS_PATTERN):]
|
| -
|
| - if len(output) > 1:
|
| - output = output.split('_')[0]
|
| -
|
| - if len(output) > 3:
|
| - contents = output.split('.')
|
| -
|
| - version = contents[2]
|
| -
|
| - if contents[3] != '0':
|
| - warningText = ('Chrome version: %s.%s but using %s.0 to bisect.' %
|
| - (version, contents[3], version))
|
| - if not warningText in self.warnings:
|
| - self.warnings.append(warningText)
|
| -
|
| - cwd = os.getcwd()
|
| - self.ChangeToDepotWorkingDirectory('chromium')
|
| - cmd = ['log', '-1', '--format=%H',
|
| - '--author=chrome-release@google.com',
|
| - '--grep=to %s' % version, 'origin/master']
|
| - return_code = bisect_utils.CheckRunGit(cmd)
|
| - os.chdir(cwd)
|
| -
|
| - results['chromium'] = output.strip()
|
| -
|
| - if depot == 'v8':
|
| - # We can't try to map the trunk revision to bleeding edge yet, because
|
| - # we don't know which direction to try to search in. Have to wait until
|
| - # the bisect has narrowed the results down to 2 v8 rolls.
|
| - results['v8_bleeding_edge'] = None
|
| -
|
| - return results
|
| -
|
| - def BackupOrRestoreOutputDirectory(self, restore=False, build_type='Release'):
|
| - """Backs up or restores build output directory based on restore argument.
|
| -
|
| - Args:
|
| - restore: Indicates whether to restore or backup. Default is False(Backup)
|
| - build_type: Target build type ('Release', 'Debug', 'Release_x64' etc.)
|
| -
|
| - Returns:
|
| - Path to backup or restored location as string. otherwise None if it fails.
|
| - """
|
| - build_dir = os.path.abspath(
|
| - builder.GetBuildOutputDirectory(self.opts, self.src_cwd))
|
| - source_dir = os.path.join(build_dir, build_type)
|
| - destination_dir = os.path.join(build_dir, '%s.bak' % build_type)
|
| - if restore:
|
| - source_dir, destination_dir = destination_dir, source_dir
|
| - if os.path.exists(source_dir):
|
| - RmTreeAndMkDir(destination_dir, skip_makedir=True)
|
| - shutil.move(source_dir, destination_dir)
|
| - return destination_dir
|
| - return None
|
| -
|
| - def GetBuildArchiveForRevision(self, revision, gs_bucket, target_arch,
|
| - patch_sha, out_dir):
|
| - """Checks and downloads build archive for a given revision.
|
| -
|
| - Checks for build archive with Git hash or SVN revision. If either of the
|
| - file exists, then downloads the archive file.
|
| -
|
| - Args:
|
| - revision: A Git hash revision.
|
| - gs_bucket: Cloud storage bucket name
|
| - target_arch: 32 or 64 bit build target
|
| - patch: A DEPS patch (used while bisecting 3rd party repositories).
|
| - out_dir: Build output directory where downloaded file is stored.
|
| -
|
| - Returns:
|
| - Downloaded archive file path if exists, otherwise None.
|
| - """
|
| - # Source archive file path on cloud storage using Git revision.
|
| - source_file = GetRemoteBuildPath(
|
| - revision, self.opts.target_platform, target_arch, patch_sha)
|
| - downloaded_archive = FetchFromCloudStorage(gs_bucket, source_file, out_dir)
|
| - if not downloaded_archive:
|
| - # Get commit position for the given SHA.
|
| - commit_position = self.source_control.GetCommitPosition(revision)
|
| - if commit_position:
|
| - # Source archive file path on cloud storage using SVN revision.
|
| - source_file = GetRemoteBuildPath(
|
| - commit_position, self.opts.target_platform, target_arch, patch_sha)
|
| - return FetchFromCloudStorage(gs_bucket, source_file, out_dir)
|
| - return downloaded_archive
|
| -
|
| - def DownloadCurrentBuild(self, revision, build_type='Release', patch=None):
|
| - """Downloads the build archive for the given revision.
|
| -
|
| - Args:
|
| - revision: The Git revision to download or build.
|
| - build_type: Target build type ('Release', 'Debug', 'Release_x64' etc.)
|
| - patch: A DEPS patch (used while bisecting 3rd party repositories).
|
| -
|
| - Returns:
|
| - True if download succeeds, otherwise False.
|
| - """
|
| - patch_sha = None
|
| - if patch:
|
| - # Get the SHA of the DEPS changes patch.
|
| - patch_sha = GetSHA1HexDigest(patch)
|
| -
|
| - # Update the DEPS changes patch with a patch to create a new file named
|
| - # 'DEPS.sha' and add patch_sha evaluated above to it.
|
| - patch = '%s\n%s' % (patch, DEPS_SHA_PATCH % {'deps_sha': patch_sha})
|
| -
|
| - # Get Build output directory
|
| - abs_build_dir = os.path.abspath(
|
| - builder.GetBuildOutputDirectory(self.opts, self.src_cwd))
|
| -
|
| - fetch_build_func = lambda: self.GetBuildArchiveForRevision(
|
| - revision, self.opts.gs_bucket, self.opts.target_arch,
|
| - patch_sha, abs_build_dir)
|
| -
|
| - # Downloaded archive file path, downloads build archive for given revision.
|
| - downloaded_file = fetch_build_func()
|
| -
|
| - # When build archive doesn't exists, post a build request to tryserver
|
| - # and wait for the build to be produced.
|
| - if not downloaded_file:
|
| - downloaded_file = self.PostBuildRequestAndWait(
|
| - revision, fetch_build=fetch_build_func, patch=patch)
|
| - if not downloaded_file:
|
| - return False
|
| -
|
| - # Generic name for the archive, created when archive file is extracted.
|
| - output_dir = os.path.join(
|
| - abs_build_dir, GetZipFileName(target_arch=self.opts.target_arch))
|
| - # Unzip build archive directory.
|
| - try:
|
| - RmTreeAndMkDir(output_dir, skip_makedir=True)
|
| - self.BackupOrRestoreOutputDirectory(restore=False)
|
| - # Build output directory based on target(e.g. out/Release, out/Debug).
|
| - target_build_output_dir = os.path.join(abs_build_dir, build_type)
|
| - ExtractZip(downloaded_file, abs_build_dir)
|
| - if not os.path.exists(output_dir):
|
| - # Due to recipe changes, the builds extract folder contains
|
| - # out/Release instead of full-build-<platform>/Release.
|
| - if os.path.exists(os.path.join(abs_build_dir, 'out', build_type)):
|
| - output_dir = os.path.join(abs_build_dir, 'out', build_type)
|
| - else:
|
| - raise IOError('Missing extracted folder %s ' % output_dir)
|
| -
|
| - print 'Moving build from %s to %s' % (
|
| - output_dir, target_build_output_dir)
|
| - shutil.move(output_dir, target_build_output_dir)
|
| - return True
|
| - except Exception as e:
|
| - print 'Something went wrong while extracting archive file: %s' % e
|
| - self.BackupOrRestoreOutputDirectory(restore=True)
|
| - # Cleanup any leftovers from unzipping.
|
| - if os.path.exists(output_dir):
|
| - RmTreeAndMkDir(output_dir, skip_makedir=True)
|
| - finally:
|
| - # Delete downloaded archive
|
| - if os.path.exists(downloaded_file):
|
| - os.remove(downloaded_file)
|
| - return False
|
| -
|
| - def PostBuildRequestAndWait(self, git_revision, fetch_build, patch=None):
|
| - """POSTs the build request job to the try server instance.
|
| -
|
| - A try job build request is posted to tryserver.chromium.perf master,
|
| - and waits for the binaries to be produced and archived on cloud storage.
|
| - Once the build is ready and stored onto cloud, build archive is downloaded
|
| - into the output folder.
|
| -
|
| - Args:
|
| - git_revision: A Git hash revision.
|
| - fetch_build: Function to check and download build from cloud storage.
|
| - patch: A DEPS patch (used while bisecting 3rd party repositories).
|
| -
|
| - Returns:
|
| - Downloaded archive file path when requested build exists and download is
|
| - successful, otherwise None.
|
| - """
|
| - def GetBuilderNameAndBuildTime(target_platform, target_arch='ia32'):
|
| - """Gets builder bot name and build time in seconds based on platform."""
|
| - # Bot names should match the one listed in tryserver.chromium's
|
| - # master.cfg which produces builds for bisect.
|
| - if bisect_utils.IsWindowsHost():
|
| - if bisect_utils.Is64BitWindows() and target_arch == 'x64':
|
| - return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME)
|
| - return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME)
|
| - if bisect_utils.IsLinuxHost():
|
| - if target_platform == 'android':
|
| - return ('android_perf_bisect_builder', MAX_LINUX_BUILD_TIME)
|
| - return ('linux_perf_bisect_builder', MAX_LINUX_BUILD_TIME)
|
| - if bisect_utils.IsMacHost():
|
| - return ('mac_perf_bisect_builder', MAX_MAC_BUILD_TIME)
|
| - raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
|
| - if not fetch_build:
|
| - return False
|
| -
|
| - bot_name, build_timeout = GetBuilderNameAndBuildTime(
|
| - self.opts.target_platform, self.opts.target_arch)
|
| - builder_host = self.opts.builder_host
|
| - builder_port = self.opts.builder_port
|
| - # Create a unique ID for each build request posted to try server builders.
|
| - # This ID is added to "Reason" property of the build.
|
| - build_request_id = GetSHA1HexDigest(
|
| - '%s-%s-%s' % (git_revision, patch, time.time()))
|
| -
|
| - # Creates a try job description.
|
| - # Always use Git hash to post build request since Commit positions are
|
| - # not supported by builders to build.
|
| - job_args = {
|
| - 'revision': 'src@%s' % git_revision,
|
| - 'bot': bot_name,
|
| - 'name': build_request_id,
|
| - }
|
| - # Update patch information if supplied.
|
| - if patch:
|
| - job_args['patch'] = patch
|
| - # Posts job to build the revision on the server.
|
| - if request_build.PostTryJob(builder_host, builder_port, job_args):
|
| - target_file, error_msg = _WaitUntilBuildIsReady(
|
| - fetch_build, bot_name, builder_host, builder_port, build_request_id,
|
| - build_timeout)
|
| - if not target_file:
|
| - print '%s [revision: %s]' % (error_msg, git_revision)
|
| - return None
|
| - return target_file
|
| - print 'Failed to post build request for revision: [%s]' % git_revision
|
| - return None
|
| -
|
| - def IsDownloadable(self, depot):
|
| - """Checks if build can be downloaded based on target platform and depot."""
|
| - if (self.opts.target_platform in ['chromium', 'android'] and
|
| - self.opts.gs_bucket):
|
| - return (depot == 'chromium' or
|
| - 'chromium' in DEPOT_DEPS_NAME[depot]['from'] or
|
| - 'v8' in DEPOT_DEPS_NAME[depot]['from'])
|
| - return False
|
| -
|
| - def UpdateDepsContents(self, deps_contents, depot, git_revision, deps_key):
|
| - """Returns modified version of DEPS file contents.
|
| -
|
| - Args:
|
| - deps_contents: DEPS file content.
|
| - depot: Current depot being bisected.
|
| - git_revision: A git hash to be updated in DEPS.
|
| - deps_key: Key in vars section of DEPS file to be searched.
|
| -
|
| - Returns:
|
| - Updated DEPS content as string if deps key is found, otherwise None.
|
| - """
|
| - # Check whether the depot and revision pattern in DEPS file vars
|
| - # e.g. for webkit the format is "webkit_revision": "12345".
|
| - deps_revision = re.compile(r'(?<="%s": ")([0-9]+)(?=")' % deps_key,
|
| - re.MULTILINE)
|
| - new_data = None
|
| - if re.search(deps_revision, deps_contents):
|
| - commit_position = self.source_control.GetCommitPosition(
|
| - git_revision, self._GetDepotDirectory(depot))
|
| - if not commit_position:
|
| - print 'Could not determine commit position for %s' % git_revision
|
| - return None
|
| - # Update the revision information for the given depot
|
| - new_data = re.sub(deps_revision, str(commit_position), deps_contents)
|
| - else:
|
| - # Check whether the depot and revision pattern in DEPS file vars
|
| - # e.g. for webkit the format is "webkit_revision": "559a6d4ab7a84c539..".
|
| - deps_revision = re.compile(
|
| - r'(?<=["\']%s["\']: ["\'])([a-fA-F0-9]{40})(?=["\'])' % deps_key,
|
| - re.MULTILINE)
|
| - if re.search(deps_revision, deps_contents):
|
| - new_data = re.sub(deps_revision, git_revision, deps_contents)
|
| - if new_data:
|
| - # For v8_bleeding_edge revisions change V8 branch in order
|
| - # to fetch bleeding edge revision.
|
| - if depot == 'v8_bleeding_edge':
|
| - new_data = _UpdateV8Branch(new_data)
|
| - if not new_data:
|
| - return None
|
| - return new_data
|
| -
|
| - def UpdateDeps(self, revision, depot, deps_file):
|
| - """Updates DEPS file with new revision of dependency repository.
|
| -
|
| - This method search DEPS for a particular pattern in which depot revision
|
| - is specified (e.g "webkit_revision": "123456"). If a match is found then
|
| - it resolves the given git hash to SVN revision and replace it in DEPS file.
|
| -
|
| - Args:
|
| - revision: A git hash revision of the dependency repository.
|
| - depot: Current depot being bisected.
|
| - deps_file: Path to DEPS file.
|
| -
|
| - Returns:
|
| - True if DEPS file is modified successfully, otherwise False.
|
| - """
|
| - if not os.path.exists(deps_file):
|
| - return False
|
| -
|
| - deps_var = DEPOT_DEPS_NAME[depot]['deps_var']
|
| - # Don't update DEPS file if deps_var is not set in DEPOT_DEPS_NAME.
|
| - if not deps_var:
|
| - print 'DEPS update not supported for Depot: %s', depot
|
| - return False
|
| -
|
| - # Hack for Angle repository. In the DEPS file, "vars" dictionary variable
|
| - # contains "angle_revision" key that holds git hash instead of SVN revision.
|
| - # And sometime "angle_revision" key is not specified in "vars" variable.
|
| - # In such cases check, "deps" dictionary variable that matches
|
| - # angle.git@[a-fA-F0-9]{40}$ and replace git hash.
|
| - if depot == 'angle':
|
| - return _UpdateDEPSForAngle(revision, depot, deps_file)
|
| -
|
| - try:
|
| - deps_contents = ReadStringFromFile(deps_file)
|
| - updated_deps_content = self.UpdateDepsContents(
|
| - deps_contents, depot, revision, deps_var)
|
| - # Write changes to DEPS file
|
| - if updated_deps_content:
|
| - WriteStringToFile(updated_deps_content, deps_file)
|
| - return True
|
| - except IOError, e:
|
| - print 'Something went wrong while updating DEPS file. [%s]' % e
|
| - return False
|
| -
|
| - def CreateDEPSPatch(self, depot, revision):
|
| - """Modifies DEPS and returns diff as text.
|
| -
|
| - Args:
|
| - depot: Current depot being bisected.
|
| - revision: A git hash revision of the dependency repository.
|
| -
|
| - Returns:
|
| - A tuple with git hash of chromium revision and DEPS patch text.
|
| - """
|
| - deps_file_path = os.path.join(self.src_cwd, bisect_utils.FILE_DEPS)
|
| - if not os.path.exists(deps_file_path):
|
| - raise RuntimeError('DEPS file does not exists.[%s]' % deps_file_path)
|
| - # Get current chromium revision (git hash).
|
| - cmd = ['rev-parse', 'HEAD']
|
| - chromium_sha = bisect_utils.CheckRunGit(cmd).strip()
|
| - if not chromium_sha:
|
| - raise RuntimeError('Failed to determine Chromium revision for %s' %
|
| - revision)
|
| - if ('chromium' in DEPOT_DEPS_NAME[depot]['from'] or
|
| - 'v8' in DEPOT_DEPS_NAME[depot]['from']):
|
| - # Checkout DEPS file for the current chromium revision.
|
| - if self.source_control.CheckoutFileAtRevision(
|
| - bisect_utils.FILE_DEPS, chromium_sha, cwd=self.src_cwd):
|
| - if self.UpdateDeps(revision, depot, deps_file_path):
|
| - diff_command = [
|
| - 'diff',
|
| - '--src-prefix=src/',
|
| - '--dst-prefix=src/',
|
| - '--no-ext-diff',
|
| - bisect_utils.FILE_DEPS,
|
| - ]
|
| - diff_text = bisect_utils.CheckRunGit(diff_command, cwd=self.src_cwd)
|
| - return (chromium_sha, ChangeBackslashToSlashInPatch(diff_text))
|
| - else:
|
| - raise RuntimeError(
|
| - 'Failed to update DEPS file for chromium: [%s]' % chromium_sha)
|
| - else:
|
| - raise RuntimeError(
|
| - 'DEPS checkout Failed for chromium revision : [%s]' % chromium_sha)
|
| - return (None, None)
|
| -
|
| - def BuildCurrentRevision(self, depot, revision=None):
|
| - """Builds chrome and performance_ui_tests on the current revision.
|
| -
|
| - Returns:
|
| - True if the build was successful.
|
| - """
|
| - if self.opts.debug_ignore_build:
|
| - return True
|
| -
|
| - build_success = False
|
| - cwd = os.getcwd()
|
| - os.chdir(self.src_cwd)
|
| - # Fetch build archive for the given revision from the cloud storage when
|
| - # the storage bucket is passed.
|
| - if self.IsDownloadable(depot) and revision:
|
| - deps_patch = None
|
| - if depot != 'chromium':
|
| - # Create a DEPS patch with new revision for dependency repository.
|
| - revision, deps_patch = self.CreateDEPSPatch(depot, revision)
|
| - if self.DownloadCurrentBuild(revision, patch=deps_patch):
|
| - if deps_patch:
|
| - # Reverts the changes to DEPS file.
|
| - self.source_control.CheckoutFileAtRevision(
|
| - bisect_utils.FILE_DEPS, revision, cwd=self.src_cwd)
|
| - build_success = True
|
| - else:
|
| - # These codes are executed when bisect bots builds binaries locally.
|
| - build_success = self.builder.Build(depot, self.opts)
|
| - os.chdir(cwd)
|
| - return build_success
|
| -
|
| - def RunGClientHooks(self):
|
| - """Runs gclient with runhooks command.
|
| -
|
| - Returns:
|
| - True if gclient reports no errors.
|
| - """
|
| - if self.opts.debug_ignore_build:
|
| - return True
|
| - return not bisect_utils.RunGClient(['runhooks'], cwd=self.src_cwd)
|
| -
|
| - def _IsBisectModeUsingMetric(self):
|
| - return self.opts.bisect_mode in [BISECT_MODE_MEAN, BISECT_MODE_STD_DEV]
|
| -
|
| - def _IsBisectModeReturnCode(self):
|
| - return self.opts.bisect_mode in [BISECT_MODE_RETURN_CODE]
|
| -
|
| - def _IsBisectModeStandardDeviation(self):
|
| - return self.opts.bisect_mode in [BISECT_MODE_STD_DEV]
|
| -
|
| - def GetCompatibleCommand(self, command_to_run, revision, depot):
|
| - # Prior to crrev.com/274857 *only* android-chromium-testshell
|
| - # Then until crrev.com/276628 *both* (android-chromium-testshell and
|
| - # android-chrome-shell) work. After that rev 276628 *only*
|
| - # android-chrome-shell works. bisect-perf-regression.py script should
|
| - # handle these cases and set appropriate browser type based on revision.
|
| - if self.opts.target_platform in ['android']:
|
| - # When its a third_party depot, get the chromium revision.
|
| - if depot != 'chromium':
|
| - revision = bisect_utils.CheckRunGit(
|
| - ['rev-parse', 'HEAD'], cwd=self.src_cwd).strip()
|
| - commit_position = self.source_control.GetCommitPosition(revision,
|
| - cwd=self.src_cwd)
|
| - if not commit_position:
|
| - return command_to_run
|
| - cmd_re = re.compile('--browser=(?P<browser_type>\S+)')
|
| - matches = cmd_re.search(command_to_run)
|
| - if bisect_utils.IsStringInt(commit_position) and matches:
|
| - cmd_browser = matches.group('browser_type')
|
| - if commit_position <= 274857 and cmd_browser == 'android-chrome-shell':
|
| - return command_to_run.replace(cmd_browser,
|
| - 'android-chromium-testshell')
|
| - elif (commit_position >= 276628 and
|
| - cmd_browser == 'android-chromium-testshell'):
|
| - return command_to_run.replace(cmd_browser,
|
| - 'android-chrome-shell')
|
| - return command_to_run
|
| -
|
| - def RunPerformanceTestAndParseResults(
|
| - self, command_to_run, metric, reset_on_first_run=False,
|
| - upload_on_last_run=False, results_label=None):
|
| - """Runs a performance test on the current revision and parses the results.
|
| -
|
| - Args:
|
| - command_to_run: The command to be run to execute the performance test.
|
| - metric: The metric to parse out from the results of the performance test.
|
| - This is the result chart name and trace name, separated by slash.
|
| - May be None for perf try jobs.
|
| - reset_on_first_run: If True, pass the flag --reset-results on first run.
|
| - upload_on_last_run: If True, pass the flag --upload-results on last run.
|
| - results_label: A value for the option flag --results-label.
|
| - The arguments reset_on_first_run, upload_on_last_run and results_label
|
| - are all ignored if the test is not a Telemetry test.
|
| -
|
| - Returns:
|
| - (values dict, 0) if --debug_ignore_perf_test was passed.
|
| - (values dict, 0, test output) if the test was run successfully.
|
| - (error message, -1) if the test couldn't be run.
|
| - (error message, -1, test output) if the test ran but there was an error.
|
| - """
|
| - success_code, failure_code = 0, -1
|
| -
|
| - if self.opts.debug_ignore_perf_test:
|
| - fake_results = {
|
| - 'mean': 0.0,
|
| - 'std_err': 0.0,
|
| - 'std_dev': 0.0,
|
| - 'values': [0.0]
|
| - }
|
| - return (fake_results, success_code)
|
| -
|
| - # For Windows platform set posix=False, to parse windows paths correctly.
|
| - # On Windows, path separators '\' or '\\' are replace by '' when posix=True,
|
| - # refer to http://bugs.python.org/issue1724822. By default posix=True.
|
| - args = shlex.split(command_to_run, posix=not bisect_utils.IsWindowsHost())
|
| -
|
| - if not _GenerateProfileIfNecessary(args):
|
| - err_text = 'Failed to generate profile for performance test.'
|
| - return (err_text, failure_code)
|
| -
|
| - # If running a Telemetry test for Chrome OS, insert the remote IP and
|
| - # identity parameters.
|
| - is_telemetry = bisect_utils.IsTelemetryCommand(command_to_run)
|
| - if self.opts.target_platform == 'cros' and is_telemetry:
|
| - args.append('--remote=%s' % self.opts.cros_remote_ip)
|
| - args.append('--identity=%s' % bisect_utils.CROS_TEST_KEY_PATH)
|
| -
|
| - start_time = time.time()
|
| -
|
| - metric_values = []
|
| - output_of_all_runs = ''
|
| - for i in xrange(self.opts.repeat_test_count):
|
| - # Can ignore the return code since if the tests fail, it won't return 0.
|
| - current_args = copy.copy(args)
|
| - if is_telemetry:
|
| - if i == 0 and reset_on_first_run:
|
| - current_args.append('--reset-results')
|
| - elif i == self.opts.repeat_test_count - 1 and upload_on_last_run:
|
| - current_args.append('--upload-results')
|
| - if results_label:
|
| - current_args.append('--results-label=%s' % results_label)
|
| - try:
|
| - output, return_code = bisect_utils.RunProcessAndRetrieveOutput(
|
| - current_args, cwd=self.src_cwd)
|
| - except OSError, e:
|
| - if e.errno == errno.ENOENT:
|
| - err_text = ('Something went wrong running the performance test. '
|
| - 'Please review the command line:\n\n')
|
| - if 'src/' in ' '.join(args):
|
| - err_text += ('Check that you haven\'t accidentally specified a '
|
| - 'path with src/ in the command.\n\n')
|
| - err_text += ' '.join(args)
|
| - err_text += '\n'
|
| -
|
| - return (err_text, failure_code)
|
| - raise
|
| -
|
| - output_of_all_runs += output
|
| - if self.opts.output_buildbot_annotations:
|
| - print output
|
| -
|
| - if metric and self._IsBisectModeUsingMetric():
|
| - metric_values += _ParseMetricValuesFromOutput(metric, output)
|
| - # If we're bisecting on a metric (ie, changes in the mean or
|
| - # standard deviation) and no metric values are produced, bail out.
|
| - if not metric_values:
|
| - break
|
| - elif self._IsBisectModeReturnCode():
|
| - metric_values.append(return_code)
|
| -
|
| - elapsed_minutes = (time.time() - start_time) / 60.0
|
| - if elapsed_minutes >= self.opts.max_time_minutes:
|
| - break
|
| -
|
| - if metric and len(metric_values) == 0:
|
| - err_text = 'Metric %s was not found in the test output.' % metric
|
| - # TODO(qyearsley): Consider also getting and displaying a list of metrics
|
| - # that were found in the output here.
|
| - return (err_text, failure_code, output_of_all_runs)
|
| -
|
| - # If we're bisecting on return codes, we're really just looking for zero vs
|
| - # non-zero.
|
| - values = {}
|
| - if self._IsBisectModeReturnCode():
|
| - # If any of the return codes is non-zero, output 1.
|
| - overall_return_code = 0 if (
|
| - all(current_value == 0 for current_value in metric_values)) else 1
|
| -
|
| - values = {
|
| - 'mean': overall_return_code,
|
| - 'std_err': 0.0,
|
| - 'std_dev': 0.0,
|
| - 'values': metric_values,
|
| - }
|
| -
|
| - print 'Results of performance test: Command returned with %d' % (
|
| - overall_return_code)
|
| - print
|
| - elif metric:
|
| - # Need to get the average value if there were multiple values.
|
| - truncated_mean = math_utils.TruncatedMean(
|
| - metric_values, self.opts.truncate_percent)
|
| - standard_err = math_utils.StandardError(metric_values)
|
| - standard_dev = math_utils.StandardDeviation(metric_values)
|
| -
|
| - if self._IsBisectModeStandardDeviation():
|
| - metric_values = [standard_dev]
|
| -
|
| - values = {
|
| - 'mean': truncated_mean,
|
| - 'std_err': standard_err,
|
| - 'std_dev': standard_dev,
|
| - 'values': metric_values,
|
| - }
|
| -
|
| - print 'Results of performance test: %12f %12f' % (
|
| - truncated_mean, standard_err)
|
| - print
|
| - return (values, success_code, output_of_all_runs)
|
| -
|
| - def _FindAllRevisionsToSync(self, revision, depot):
|
| - """Finds all dependent revisions and depots that need to be synced.
|
| -
|
| - For example skia is broken up into 3 git mirrors over skia/src,
|
| - skia/gyp, and skia/include. To sync skia/src properly, one has to find
|
| - the proper revisions in skia/gyp and skia/include.
|
| -
|
| - This is only useful in the git workflow, as an SVN depot may be split into
|
| - multiple mirrors.
|
| -
|
| - Args:
|
| - revision: The revision to sync to.
|
| - depot: The depot in use at the moment (probably skia).
|
| -
|
| - Returns:
|
| - A list of [depot, revision] pairs that need to be synced.
|
| - """
|
| - revisions_to_sync = [[depot, revision]]
|
| -
|
| - is_base = ((depot == 'chromium') or (depot == 'cros') or
|
| - (depot == 'android-chrome'))
|
| -
|
| - # Some SVN depots were split into multiple git depots, so we need to
|
| - # figure out for each mirror which git revision to grab. There's no
|
| - # guarantee that the SVN revision will exist for each of the dependent
|
| - # depots, so we have to grep the git logs and grab the next earlier one.
|
| - if (not is_base
|
| - and DEPOT_DEPS_NAME[depot]['depends']
|
| - and self.source_control.IsGit()):
|
| - commit_position = self.source_control.GetCommitPosition(revision)
|
| -
|
| - for d in DEPOT_DEPS_NAME[depot]['depends']:
|
| - self.ChangeToDepotWorkingDirectory(d)
|
| -
|
| - dependant_rev = self.source_control.ResolveToRevision(
|
| - commit_position, d, DEPOT_DEPS_NAME, -1000)
|
| -
|
| - if dependant_rev:
|
| - revisions_to_sync.append([d, dependant_rev])
|
| -
|
| - num_resolved = len(revisions_to_sync)
|
| - num_needed = len(DEPOT_DEPS_NAME[depot]['depends'])
|
| -
|
| - self.ChangeToDepotWorkingDirectory(depot)
|
| -
|
| - if not ((num_resolved - 1) == num_needed):
|
| - return None
|
| -
|
| - return revisions_to_sync
|
| -
|
| - def PerformPreBuildCleanup(self):
|
| - """Performs cleanup between runs."""
|
| - print 'Cleaning up between runs.'
|
| - print
|
| -
|
| - # Leaving these .pyc files around between runs may disrupt some perf tests.
|
| - for (path, _, files) in os.walk(self.src_cwd):
|
| - for cur_file in files:
|
| - if cur_file.endswith('.pyc'):
|
| - path_to_file = os.path.join(path, cur_file)
|
| - os.remove(path_to_file)
|
| -
|
| - def PerformCrosChrootCleanup(self):
|
| - """Deletes the chroot.
|
| -
|
| - Returns:
|
| - True if successful.
|
| - """
|
| - cwd = os.getcwd()
|
| - self.ChangeToDepotWorkingDirectory('cros')
|
| - cmd = [bisect_utils.CROS_SDK_PATH, '--delete']
|
| - return_code = bisect_utils.RunProcess(cmd)
|
| - os.chdir(cwd)
|
| - return not return_code
|
| -
|
| - def CreateCrosChroot(self):
|
| - """Creates a new chroot.
|
| -
|
| - Returns:
|
| - True if successful.
|
| - """
|
| - cwd = os.getcwd()
|
| - self.ChangeToDepotWorkingDirectory('cros')
|
| - cmd = [bisect_utils.CROS_SDK_PATH, '--create']
|
| - return_code = bisect_utils.RunProcess(cmd)
|
| - os.chdir(cwd)
|
| - return not return_code
|
| -
|
| - def _PerformPreSyncCleanup(self, depot):
|
| - """Performs any necessary cleanup before syncing.
|
| -
|
| - Args:
|
| - depot: Depot name.
|
| -
|
| - Returns:
|
| - True if successful.
|
| - """
|
| - if depot == 'chromium' or depot == 'android-chrome':
|
| - # Removes third_party/libjingle. At some point, libjingle was causing
|
| - # issues syncing when using the git workflow (crbug.com/266324).
|
| - os.chdir(self.src_cwd)
|
| - if not bisect_utils.RemoveThirdPartyDirectory('libjingle'):
|
| - return False
|
| - # Removes third_party/skia. At some point, skia was causing
|
| - # issues syncing when using the git workflow (crbug.com/377951).
|
| - if not bisect_utils.RemoveThirdPartyDirectory('skia'):
|
| - return False
|
| - elif depot == 'cros':
|
| - return self.PerformCrosChrootCleanup()
|
| - return True
|
| -
|
| - def _RunPostSync(self, depot):
|
| - """Performs any work after syncing.
|
| -
|
| - Args:
|
| - depot: Depot name.
|
| -
|
| - Returns:
|
| - True if successful.
|
| - """
|
| - if self.opts.target_platform == 'android':
|
| - if not builder.SetupAndroidBuildEnvironment(self.opts,
|
| - path_to_src=self.src_cwd):
|
| - return False
|
| -
|
| - if depot == 'cros':
|
| - return self.CreateCrosChroot()
|
| - else:
|
| - return self.RunGClientHooks()
|
| - return True
|
| -
|
| - def ShouldSkipRevision(self, depot, revision):
|
| - """Checks whether a particular revision can be safely skipped.
|
| -
|
| - Some commits can be safely skipped (such as a DEPS roll), since the tool
|
| - is git based those changes would have no effect.
|
| -
|
| - Args:
|
| - depot: The depot being bisected.
|
| - revision: Current revision we're synced to.
|
| -
|
| - Returns:
|
| - True if we should skip building/testing this revision.
|
| - """
|
| - if depot == 'chromium':
|
| - if self.source_control.IsGit():
|
| - cmd = ['diff-tree', '--no-commit-id', '--name-only', '-r', revision]
|
| - output = bisect_utils.CheckRunGit(cmd)
|
| -
|
| - files = output.splitlines()
|
| -
|
| - if len(files) == 1 and files[0] == 'DEPS':
|
| - return True
|
| -
|
| - return False
|
| -
|
| - def RunTest(self, revision, depot, command, metric, skippable=False):
|
| - """Performs a full sync/build/run of the specified revision.
|
| -
|
| - Args:
|
| - revision: The revision to sync to.
|
| - depot: The depot that's being used at the moment (src, webkit, etc.)
|
| - command: The command to execute the performance test.
|
| - metric: The performance metric being tested.
|
| -
|
| - Returns:
|
| - On success, a tuple containing the results of the performance test.
|
| - Otherwise, a tuple with the error message.
|
| - """
|
| - # Decide which sync program to use.
|
| - sync_client = None
|
| - if depot == 'chromium' or depot == 'android-chrome':
|
| - sync_client = 'gclient'
|
| - elif depot == 'cros':
|
| - sync_client = 'repo'
|
| -
|
| - # Decide what depots will need to be synced to what revisions.
|
| - revisions_to_sync = self._FindAllRevisionsToSync(revision, depot)
|
| - if not revisions_to_sync:
|
| - return ('Failed to resolve dependent depots.', BUILD_RESULT_FAIL)
|
| -
|
| - if not self._PerformPreSyncCleanup(depot):
|
| - return ('Failed to perform pre-sync cleanup.', BUILD_RESULT_FAIL)
|
| -
|
| - # Do the syncing for all depots.
|
| - if not self.opts.debug_ignore_sync:
|
| - if not self._SyncAllRevisions(revisions_to_sync, sync_client):
|
| - return ('Failed to sync: [%s]' % str(revision), BUILD_RESULT_FAIL)
|
| -
|
| - # Try to do any post-sync steps. This may include "gclient runhooks".
|
| - if not self._RunPostSync(depot):
|
| - return ('Failed to run [gclient runhooks].', BUILD_RESULT_FAIL)
|
| -
|
| - # Skip this revision if it can be skipped.
|
| - if skippable and self.ShouldSkipRevision(depot, revision):
|
| - return ('Skipped revision: [%s]' % str(revision),
|
| - BUILD_RESULT_SKIPPED)
|
| -
|
| - # Obtain a build for this revision. This may be done by requesting a build
|
| - # from another builder, waiting for it and downloading it.
|
| - start_build_time = time.time()
|
| - build_success = self.BuildCurrentRevision(depot, revision)
|
| - if not build_success:
|
| - return ('Failed to build revision: [%s]' % str(revision),
|
| - BUILD_RESULT_FAIL)
|
| - after_build_time = time.time()
|
| -
|
| - # Possibly alter the command.
|
| - command = self.GetCompatibleCommand(command, revision, depot)
|
| -
|
| - # Run the command and get the results.
|
| - results = self.RunPerformanceTestAndParseResults(command, metric)
|
| -
|
| - # Restore build output directory once the tests are done, to avoid
|
| - # any discrepancies.
|
| - if self.IsDownloadable(depot) and revision:
|
| - self.BackupOrRestoreOutputDirectory(restore=True)
|
| -
|
| - # A value other than 0 indicates that the test couldn't be run, and results
|
| - # should also include an error message.
|
| - if results[1] != 0:
|
| - return results
|
| -
|
| - external_revisions = self._Get3rdPartyRevisions(depot)
|
| -
|
| - if not external_revisions is None:
|
| - return (results[0], results[1], external_revisions,
|
| - time.time() - after_build_time, after_build_time -
|
| - start_build_time)
|
| - else:
|
| - return ('Failed to parse DEPS file for external revisions.',
|
| - BUILD_RESULT_FAIL)
|
| -
|
| - def _SyncAllRevisions(self, revisions_to_sync, sync_client):
|
| - """Syncs multiple depots to particular revisions.
|
| -
|
| - Args:
|
| - revisions_to_sync: A list of (depot, revision) pairs to be synced.
|
| - sync_client: Program used to sync, e.g. "gclient", "repo". Can be None.
|
| -
|
| - Returns:
|
| - True if successful, False otherwise.
|
| - """
|
| - for depot, revision in revisions_to_sync:
|
| - self.ChangeToDepotWorkingDirectory(depot)
|
| -
|
| - if sync_client:
|
| - self.PerformPreBuildCleanup()
|
| -
|
| - # When using gclient to sync, you need to specify the depot you
|
| - # want so that all the dependencies sync properly as well.
|
| - # i.e. gclient sync src@<SHA1>
|
| - if sync_client == 'gclient':
|
| - revision = '%s@%s' % (DEPOT_DEPS_NAME[depot]['src'], revision)
|
| -
|
| - sync_success = self.source_control.SyncToRevision(revision, sync_client)
|
| - if not sync_success:
|
| - return False
|
| -
|
| - return True
|
| -
|
| - def _CheckIfRunPassed(self, current_value, known_good_value, known_bad_value):
|
| - """Given known good and bad values, decide if the current_value passed
|
| - or failed.
|
| -
|
| - Args:
|
| - current_value: The value of the metric being checked.
|
| - known_bad_value: The reference value for a "failed" run.
|
| - known_good_value: The reference value for a "passed" run.
|
| -
|
| - Returns:
|
| - True if the current_value is closer to the known_good_value than the
|
| - known_bad_value.
|
| - """
|
| - if self.opts.bisect_mode == BISECT_MODE_STD_DEV:
|
| - dist_to_good_value = abs(current_value['std_dev'] -
|
| - known_good_value['std_dev'])
|
| - dist_to_bad_value = abs(current_value['std_dev'] -
|
| - known_bad_value['std_dev'])
|
| - else:
|
| - dist_to_good_value = abs(current_value['mean'] - known_good_value['mean'])
|
| - dist_to_bad_value = abs(current_value['mean'] - known_bad_value['mean'])
|
| -
|
| - return dist_to_good_value < dist_to_bad_value
|
| -
|
| - def _GetDepotDirectory(self, depot_name):
|
| - if depot_name == 'chromium':
|
| - return self.src_cwd
|
| - elif depot_name == 'cros':
|
| - return self.cros_cwd
|
| - elif depot_name in DEPOT_NAMES:
|
| - return self.depot_cwd[depot_name]
|
| - else:
|
| - assert False, ('Unknown depot [ %s ] encountered. Possibly a new one '
|
| - 'was added without proper support?' % depot_name)
|
| -
|
| - def ChangeToDepotWorkingDirectory(self, depot_name):
|
| - """Given a depot, changes to the appropriate working directory.
|
| -
|
| - Args:
|
| - depot_name: The name of the depot (see DEPOT_NAMES).
|
| - """
|
| - os.chdir(self._GetDepotDirectory(depot_name))
|
| -
|
| - def _FillInV8BleedingEdgeInfo(self, min_revision_data, max_revision_data):
|
| - r1 = self._GetNearestV8BleedingEdgeFromTrunk(min_revision_data['revision'],
|
| - search_forward=True)
|
| - r2 = self._GetNearestV8BleedingEdgeFromTrunk(max_revision_data['revision'],
|
| - search_forward=False)
|
| - min_revision_data['external']['v8_bleeding_edge'] = r1
|
| - max_revision_data['external']['v8_bleeding_edge'] = r2
|
| -
|
| - if (not self._GetV8BleedingEdgeFromV8TrunkIfMappable(
|
| - min_revision_data['revision'])
|
| - or not self._GetV8BleedingEdgeFromV8TrunkIfMappable(
|
| - max_revision_data['revision'])):
|
| - self.warnings.append(
|
| - 'Trunk revisions in V8 did not map directly to bleeding_edge. '
|
| - 'Attempted to expand the range to find V8 rolls which did map '
|
| - 'directly to bleeding_edge revisions, but results might not be '
|
| - 'valid.')
|
| -
|
| - def _FindNextDepotToBisect(
|
| - self, current_depot, min_revision_data, max_revision_data):
|
| - """Decides which depot the script should dive into next (if any).
|
| -
|
| - Args:
|
| - current_depot: Current depot being bisected.
|
| - min_revision_data: Data about the earliest revision in the bisect range.
|
| - max_revision_data: Data about the latest revision in the bisect range.
|
| -
|
| - Returns:
|
| - Name of the depot to bisect next, or None.
|
| - """
|
| - external_depot = None
|
| - for next_depot in DEPOT_NAMES:
|
| - if DEPOT_DEPS_NAME[next_depot].has_key('platform'):
|
| - if DEPOT_DEPS_NAME[next_depot]['platform'] != os.name:
|
| - continue
|
| -
|
| - if not (DEPOT_DEPS_NAME[next_depot]['recurse']
|
| - and min_revision_data['depot']
|
| - in DEPOT_DEPS_NAME[next_depot]['from']):
|
| - continue
|
| -
|
| - if current_depot == 'v8':
|
| - # We grab the bleeding_edge info here rather than earlier because we
|
| - # finally have the revision range. From that we can search forwards and
|
| - # backwards to try to match trunk revisions to bleeding_edge.
|
| - self._FillInV8BleedingEdgeInfo(min_revision_data, max_revision_data)
|
| -
|
| - if (min_revision_data['external'].get(next_depot) ==
|
| - max_revision_data['external'].get(next_depot)):
|
| - continue
|
| -
|
| - if (min_revision_data['external'].get(next_depot) and
|
| - max_revision_data['external'].get(next_depot)):
|
| - external_depot = next_depot
|
| - break
|
| -
|
| - return external_depot
|
| -
|
| - def PrepareToBisectOnDepot(
|
| - self, current_depot, end_revision, start_revision, previous_revision):
|
| - """Changes to the appropriate directory and gathers a list of revisions
|
| - to bisect between |start_revision| and |end_revision|.
|
| -
|
| - Args:
|
| - current_depot: The depot we want to bisect.
|
| - end_revision: End of the revision range.
|
| - start_revision: Start of the revision range.
|
| - previous_revision: The last revision we synced to on |previous_depot|.
|
| -
|
| - Returns:
|
| - A list containing the revisions between |start_revision| and
|
| - |end_revision| inclusive.
|
| - """
|
| - # Change into working directory of external library to run
|
| - # subsequent commands.
|
| - self.ChangeToDepotWorkingDirectory(current_depot)
|
| -
|
| - # V8 (and possibly others) is merged in periodically. Bisecting
|
| - # this directory directly won't give much good info.
|
| - if DEPOT_DEPS_NAME[current_depot].has_key('custom_deps'):
|
| - config_path = os.path.join(self.src_cwd, '..')
|
| - if bisect_utils.RunGClientAndCreateConfig(self.opts,
|
| - DEPOT_DEPS_NAME[current_depot]['custom_deps'], cwd=config_path):
|
| - return []
|
| - if bisect_utils.RunGClient(
|
| - ['sync', '--revision', previous_revision], cwd=self.src_cwd):
|
| - return []
|
| -
|
| - if current_depot == 'v8_bleeding_edge':
|
| - self.ChangeToDepotWorkingDirectory('chromium')
|
| -
|
| - shutil.move('v8', 'v8.bak')
|
| - shutil.move('v8_bleeding_edge', 'v8')
|
| -
|
| - self.cleanup_commands.append(['mv', 'v8', 'v8_bleeding_edge'])
|
| - self.cleanup_commands.append(['mv', 'v8.bak', 'v8'])
|
| -
|
| - self.depot_cwd['v8_bleeding_edge'] = os.path.join(self.src_cwd, 'v8')
|
| - self.depot_cwd['v8'] = os.path.join(self.src_cwd, 'v8.bak')
|
| -
|
| - self.ChangeToDepotWorkingDirectory(current_depot)
|
| -
|
| - depot_revision_list = self.GetRevisionList(current_depot,
|
| - end_revision,
|
| - start_revision)
|
| -
|
| - self.ChangeToDepotWorkingDirectory('chromium')
|
| -
|
| - return depot_revision_list
|
| -
|
| - def GatherReferenceValues(self, good_rev, bad_rev, cmd, metric, target_depot):
|
| - """Gathers reference values by running the performance tests on the
|
| - known good and bad revisions.
|
| -
|
| - Args:
|
| - good_rev: The last known good revision where the performance regression
|
| - has not occurred yet.
|
| - bad_rev: A revision where the performance regression has already occurred.
|
| - cmd: The command to execute the performance test.
|
| - metric: The metric being tested for regression.
|
| -
|
| - Returns:
|
| - A tuple with the results of building and running each revision.
|
| - """
|
| - bad_run_results = self.RunTest(bad_rev, target_depot, cmd, metric)
|
| -
|
| - good_run_results = None
|
| -
|
| - if not bad_run_results[1]:
|
| - good_run_results = self.RunTest(good_rev, target_depot, cmd, metric)
|
| -
|
| - return (bad_run_results, good_run_results)
|
| -
|
| - def PrintRevisionsToBisectMessage(self, revision_list, depot):
|
| - if self.opts.output_buildbot_annotations:
|
| - step_name = 'Bisection Range: [%s - %s]' % (
|
| - revision_list[len(revision_list)-1], revision_list[0])
|
| - bisect_utils.OutputAnnotationStepStart(step_name)
|
| -
|
| - print
|
| - print 'Revisions to bisect on [%s]:' % depot
|
| - for revision_id in revision_list:
|
| - print ' -> %s' % (revision_id, )
|
| - print
|
| -
|
| - if self.opts.output_buildbot_annotations:
|
| - bisect_utils.OutputAnnotationStepClosed()
|
| -
|
| - def NudgeRevisionsIfDEPSChange(self, bad_revision, good_revision,
|
| - good_svn_revision=None):
|
| - """Checks to see if changes to DEPS file occurred, and that the revision
|
| - range also includes the change to .DEPS.git. If it doesn't, attempts to
|
| - expand the revision range to include it.
|
| -
|
| - Args:
|
| - bad_revision: First known bad git revision.
|
| - good_revision: Last known good git revision.
|
| - good_svn_revision: Last known good svn revision.
|
| -
|
| - Returns:
|
| - A tuple with the new bad and good revisions.
|
| - """
|
| - # DONOT perform nudge because at revision 291563 .DEPS.git was removed
|
| - # and source contain only DEPS file for dependency changes.
|
| - if good_svn_revision >= 291563:
|
| - return (bad_revision, good_revision)
|
| -
|
| - if self.source_control.IsGit() and self.opts.target_platform == 'chromium':
|
| - changes_to_deps = self.source_control.QueryFileRevisionHistory(
|
| - bisect_utils.FILE_DEPS, good_revision, bad_revision)
|
| -
|
| - if changes_to_deps:
|
| - # DEPS file was changed, search from the oldest change to DEPS file to
|
| - # bad_revision to see if there are matching .DEPS.git changes.
|
| - oldest_deps_change = changes_to_deps[-1]
|
| - changes_to_gitdeps = self.source_control.QueryFileRevisionHistory(
|
| - bisect_utils.FILE_DEPS_GIT, oldest_deps_change, bad_revision)
|
| -
|
| - if len(changes_to_deps) != len(changes_to_gitdeps):
|
| - # Grab the timestamp of the last DEPS change
|
| - cmd = ['log', '--format=%ct', '-1', changes_to_deps[0]]
|
| - output = bisect_utils.CheckRunGit(cmd)
|
| - commit_time = int(output)
|
| -
|
| - # Try looking for a commit that touches the .DEPS.git file in the
|
| - # next 15 minutes after the DEPS file change.
|
| - cmd = ['log', '--format=%H', '-1',
|
| - '--before=%d' % (commit_time + 900), '--after=%d' % commit_time,
|
| - 'origin/master', '--', bisect_utils.FILE_DEPS_GIT]
|
| - output = bisect_utils.CheckRunGit(cmd)
|
| - output = output.strip()
|
| - if output:
|
| - self.warnings.append('Detected change to DEPS and modified '
|
| - 'revision range to include change to .DEPS.git')
|
| - return (output, good_revision)
|
| - else:
|
| - self.warnings.append('Detected change to DEPS but couldn\'t find '
|
| - 'matching change to .DEPS.git')
|
| - return (bad_revision, good_revision)
|
| -
|
| - def CheckIfRevisionsInProperOrder(
|
| - self, target_depot, good_revision, bad_revision):
|
| - """Checks that |good_revision| is an earlier revision than |bad_revision|.
|
| -
|
| - Args:
|
| - good_revision: Number/tag of the known good revision.
|
| - bad_revision: Number/tag of the known bad revision.
|
| -
|
| - Returns:
|
| - True if the revisions are in the proper order (good earlier than bad).
|
| - """
|
| - if self.source_control.IsGit() and target_depot != 'cros':
|
| - cwd = self._GetDepotDirectory(target_depot)
|
| -
|
| - cmd = ['log', '--format=%ct', '-1', good_revision]
|
| - output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
|
| - good_commit_time = int(output)
|
| -
|
| - cmd = ['log', '--format=%ct', '-1', bad_revision]
|
| - output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
|
| - bad_commit_time = int(output)
|
| -
|
| - return good_commit_time <= bad_commit_time
|
| - else:
|
| - # CrOS and SVN use integers.
|
| - return int(good_revision) <= int(bad_revision)
|
| -
|
| - def CanPerformBisect(self, good_revision, bad_revision):
|
| - """Checks whether a given revision is bisectable.
|
| -
|
| - Checks for following:
|
| - 1. Non-bisectable revsions for android bots (refer to crbug.com/385324).
|
| - 2. Non-bisectable revsions for Windows bots (refer to crbug.com/405274).
|
| -
|
| - Args:
|
| - good_revision: Known good revision.
|
| - bad_revision: Known bad revision.
|
| -
|
| - Returns:
|
| - A dictionary indicating the result. If revision is not bisectable,
|
| - this will contain the field "error", otherwise None.
|
| - """
|
| - if self.opts.target_platform == 'android':
|
| - revision_to_check = self.source_control.GetCommitPosition(good_revision)
|
| - if (bisect_utils.IsStringInt(good_revision)
|
| - and good_revision < 265549):
|
| - return {'error': (
|
| - 'Bisect cannot continue for the given revision range.\n'
|
| - 'It is impossible to bisect Android regressions '
|
| - 'prior to r265549, which allows the bisect bot to '
|
| - 'rely on Telemetry to do apk installation of the most recently '
|
| - 'built local ChromeShell(refer to crbug.com/385324).\n'
|
| - 'Please try bisecting revisions greater than or equal to r265549.')}
|
| -
|
| - if bisect_utils.IsWindowsHost():
|
| - good_revision = self.source_control.GetCommitPosition(good_revision)
|
| - bad_revision = self.source_control.GetCommitPosition(bad_revision)
|
| - if (bisect_utils.IsStringInt(good_revision) and
|
| - bisect_utils.IsStringInt(bad_revision)):
|
| - if (289987 <= good_revision < 290716 or
|
| - 289987 <= bad_revision < 290716):
|
| - return {'error': ('Oops! Revision between r289987 and r290716 are '
|
| - 'marked as dead zone for Windows due to '
|
| - 'crbug.com/405274. Please try another range.')}
|
| -
|
| - return None
|
| -
|
| - def Run(self, command_to_run, bad_revision_in, good_revision_in, metric):
|
| - """Given known good and bad revisions, run a binary search on all
|
| - intermediate revisions to determine the CL where the performance regression
|
| - occurred.
|
| -
|
| - Args:
|
| - command_to_run: Specify the command to execute the performance test.
|
| - good_revision: Number/tag of the known good revision.
|
| - bad_revision: Number/tag of the known bad revision.
|
| - metric: The performance metric to monitor.
|
| -
|
| - Returns:
|
| - A dict with 2 members, 'revision_data' and 'error'. On success,
|
| - 'revision_data' will contain a dict mapping revision ids to
|
| - data about that revision. Each piece of revision data consists of a
|
| - dict with the following keys:
|
| -
|
| - 'passed': Represents whether the performance test was successful at
|
| - that revision. Possible values include: 1 (passed), 0 (failed),
|
| - '?' (skipped), 'F' (build failed).
|
| - 'depot': The depot that this revision is from (i.e. WebKit)
|
| - 'external': If the revision is a 'src' revision, 'external' contains
|
| - the revisions of each of the external libraries.
|
| - 'sort': A sort value for sorting the dict in order of commits.
|
| -
|
| - For example:
|
| - {
|
| - 'error':None,
|
| - 'revision_data':
|
| - {
|
| - 'CL #1':
|
| - {
|
| - 'passed': False,
|
| - 'depot': 'chromium',
|
| - 'external': None,
|
| - 'sort': 0
|
| - }
|
| - }
|
| - }
|
| -
|
| - If an error occurred, the 'error' field will contain the message and
|
| - 'revision_data' will be empty.
|
| - """
|
| - results = {
|
| - 'revision_data' : {},
|
| - 'error' : None,
|
| - }
|
| -
|
| - # Choose depot to bisect first
|
| - target_depot = 'chromium'
|
| - if self.opts.target_platform == 'cros':
|
| - target_depot = 'cros'
|
| - elif self.opts.target_platform == 'android-chrome':
|
| - target_depot = 'android-chrome'
|
| -
|
| - cwd = os.getcwd()
|
| - self.ChangeToDepotWorkingDirectory(target_depot)
|
| -
|
| - # If they passed SVN revisions, we can try match them to git SHA1 hashes.
|
| - bad_revision = self.source_control.ResolveToRevision(
|
| - bad_revision_in, target_depot, DEPOT_DEPS_NAME, 100)
|
| - good_revision = self.source_control.ResolveToRevision(
|
| - good_revision_in, target_depot, DEPOT_DEPS_NAME, -100)
|
| -
|
| - os.chdir(cwd)
|
| - if bad_revision is None:
|
| - results['error'] = 'Couldn\'t resolve [%s] to SHA1.' % bad_revision_in
|
| - return results
|
| -
|
| - if good_revision is None:
|
| - results['error'] = 'Couldn\'t resolve [%s] to SHA1.' % good_revision_in
|
| - return results
|
| -
|
| - # Check that they didn't accidentally swap good and bad revisions.
|
| - if not self.CheckIfRevisionsInProperOrder(
|
| - target_depot, good_revision, bad_revision):
|
| - results['error'] = ('bad_revision < good_revision, did you swap these '
|
| - 'by mistake?')
|
| - return results
|
| - bad_revision, good_revision = self.NudgeRevisionsIfDEPSChange(
|
| - bad_revision, good_revision, good_revision_in)
|
| - if self.opts.output_buildbot_annotations:
|
| - bisect_utils.OutputAnnotationStepStart('Gathering Revisions')
|
| -
|
| - cannot_bisect = self.CanPerformBisect(good_revision, bad_revision)
|
| - if cannot_bisect:
|
| - results['error'] = cannot_bisect.get('error')
|
| - return results
|
| -
|
| - print 'Gathering revision range for bisection.'
|
| - # Retrieve a list of revisions to do bisection on.
|
| - src_revision_list = self.GetRevisionList(
|
| - target_depot, bad_revision, good_revision)
|
| -
|
| - if self.opts.output_buildbot_annotations:
|
| - bisect_utils.OutputAnnotationStepClosed()
|
| -
|
| - if src_revision_list:
|
| - # revision_data will store information about a revision such as the
|
| - # depot it came from, the webkit/V8 revision at that time,
|
| - # performance timing, build state, etc...
|
| - revision_data = results['revision_data']
|
| -
|
| - # revision_list is the list we're binary searching through at the moment.
|
| - revision_list = []
|
| -
|
| - sort_key_ids = 0
|
| -
|
| - for current_revision_id in src_revision_list:
|
| - sort_key_ids += 1
|
| -
|
| - revision_data[current_revision_id] = {
|
| - 'value' : None,
|
| - 'passed' : '?',
|
| - 'depot' : target_depot,
|
| - 'external' : None,
|
| - 'perf_time' : 0,
|
| - 'build_time' : 0,
|
| - 'sort' : sort_key_ids,
|
| - }
|
| - revision_list.append(current_revision_id)
|
| -
|
| - min_revision = 0
|
| - max_revision = len(revision_list) - 1
|
| -
|
| - self.PrintRevisionsToBisectMessage(revision_list, target_depot)
|
| -
|
| - if self.opts.output_buildbot_annotations:
|
| - bisect_utils.OutputAnnotationStepStart('Gathering Reference Values')
|
| -
|
| - print 'Gathering reference values for bisection.'
|
| -
|
| - # Perform the performance tests on the good and bad revisions, to get
|
| - # reference values.
|
| - bad_results, good_results = self.GatherReferenceValues(good_revision,
|
| - bad_revision,
|
| - command_to_run,
|
| - metric,
|
| - target_depot)
|
| -
|
| - if self.opts.output_buildbot_annotations:
|
| - bisect_utils.OutputAnnotationStepClosed()
|
| -
|
| - if bad_results[1]:
|
| - results['error'] = ('An error occurred while building and running '
|
| - 'the \'bad\' reference value. The bisect cannot continue without '
|
| - 'a working \'bad\' revision to start from.\n\nError: %s' %
|
| - bad_results[0])
|
| - return results
|
| -
|
| - if good_results[1]:
|
| - results['error'] = ('An error occurred while building and running '
|
| - 'the \'good\' reference value. The bisect cannot continue without '
|
| - 'a working \'good\' revision to start from.\n\nError: %s' %
|
| - good_results[0])
|
| - return results
|
| -
|
| -
|
| - # We need these reference values to determine if later runs should be
|
| - # classified as pass or fail.
|
| - known_bad_value = bad_results[0]
|
| - known_good_value = good_results[0]
|
| -
|
| - # Can just mark the good and bad revisions explicitly here since we
|
| - # already know the results.
|
| - bad_revision_data = revision_data[revision_list[0]]
|
| - bad_revision_data['external'] = bad_results[2]
|
| - bad_revision_data['perf_time'] = bad_results[3]
|
| - bad_revision_data['build_time'] = bad_results[4]
|
| - bad_revision_data['passed'] = False
|
| - bad_revision_data['value'] = known_bad_value
|
| -
|
| - good_revision_data = revision_data[revision_list[max_revision]]
|
| - good_revision_data['external'] = good_results[2]
|
| - good_revision_data['perf_time'] = good_results[3]
|
| - good_revision_data['build_time'] = good_results[4]
|
| - good_revision_data['passed'] = True
|
| - good_revision_data['value'] = known_good_value
|
| -
|
| - next_revision_depot = target_depot
|
| -
|
| - while True:
|
| - if not revision_list:
|
| - break
|
| -
|
| - min_revision_data = revision_data[revision_list[min_revision]]
|
| - max_revision_data = revision_data[revision_list[max_revision]]
|
| -
|
| - if max_revision - min_revision <= 1:
|
| - current_depot = min_revision_data['depot']
|
| - if min_revision_data['passed'] == '?':
|
| - next_revision_index = min_revision
|
| - elif max_revision_data['passed'] == '?':
|
| - next_revision_index = max_revision
|
| - elif current_depot in ['android-chrome', 'cros', 'chromium', 'v8']:
|
| - previous_revision = revision_list[min_revision]
|
| - # If there were changes to any of the external libraries we track,
|
| - # should bisect the changes there as well.
|
| - external_depot = self._FindNextDepotToBisect(
|
| - current_depot, min_revision_data, max_revision_data)
|
| - # If there was no change in any of the external depots, the search
|
| - # is over.
|
| - if not external_depot:
|
| - if current_depot == 'v8':
|
| - self.warnings.append('Unfortunately, V8 bisection couldn\'t '
|
| - 'continue any further. The script can only bisect into '
|
| - 'V8\'s bleeding_edge repository if both the current and '
|
| - 'previous revisions in trunk map directly to revisions in '
|
| - 'bleeding_edge.')
|
| - break
|
| -
|
| - earliest_revision = max_revision_data['external'][external_depot]
|
| - latest_revision = min_revision_data['external'][external_depot]
|
| -
|
| - new_revision_list = self.PrepareToBisectOnDepot(
|
| - external_depot, latest_revision, earliest_revision,
|
| - previous_revision)
|
| -
|
| - if not new_revision_list:
|
| - results['error'] = ('An error occurred attempting to retrieve '
|
| - 'revision range: [%s..%s]' %
|
| - (earliest_revision, latest_revision))
|
| - return results
|
| -
|
| - _AddRevisionsIntoRevisionData(
|
| - new_revision_list, external_depot, min_revision_data['sort'],
|
| - revision_data)
|
| -
|
| - # Reset the bisection and perform it on the newly inserted
|
| - # changelists.
|
| - revision_list = new_revision_list
|
| - min_revision = 0
|
| - max_revision = len(revision_list) - 1
|
| - sort_key_ids += len(revision_list)
|
| -
|
| - print ('Regression in metric %s appears to be the result of '
|
| - 'changes in [%s].' % (metric, external_depot))
|
| -
|
| - self.PrintRevisionsToBisectMessage(revision_list, external_depot)
|
| -
|
| - continue
|
| - else:
|
| - break
|
| - else:
|
| - next_revision_index = (int((max_revision - min_revision) / 2) +
|
| - min_revision)
|
| -
|
| - next_revision_id = revision_list[next_revision_index]
|
| - next_revision_data = revision_data[next_revision_id]
|
| - next_revision_depot = next_revision_data['depot']
|
| -
|
| - self.ChangeToDepotWorkingDirectory(next_revision_depot)
|
| -
|
| - if self.opts.output_buildbot_annotations:
|
| - step_name = 'Working on [%s]' % next_revision_id
|
| - bisect_utils.OutputAnnotationStepStart(step_name)
|
| -
|
| - print 'Working on revision: [%s]' % next_revision_id
|
| -
|
| - run_results = self.RunTest(
|
| - next_revision_id, next_revision_depot, command_to_run, metric,
|
| - skippable=True)
|
| -
|
| - # If the build is successful, check whether or not the metric
|
| - # had regressed.
|
| - if not run_results[1]:
|
| - if len(run_results) > 2:
|
| - next_revision_data['external'] = run_results[2]
|
| - next_revision_data['perf_time'] = run_results[3]
|
| - next_revision_data['build_time'] = run_results[4]
|
| -
|
| - passed_regression = self._CheckIfRunPassed(run_results[0],
|
| - known_good_value,
|
| - known_bad_value)
|
| -
|
| - next_revision_data['passed'] = passed_regression
|
| - next_revision_data['value'] = run_results[0]
|
| -
|
| - if passed_regression:
|
| - max_revision = next_revision_index
|
| - else:
|
| - min_revision = next_revision_index
|
| - else:
|
| - if run_results[1] == BUILD_RESULT_SKIPPED:
|
| - next_revision_data['passed'] = 'Skipped'
|
| - elif run_results[1] == BUILD_RESULT_FAIL:
|
| - next_revision_data['passed'] = 'Build Failed'
|
| -
|
| - print run_results[0]
|
| -
|
| - # If the build is broken, remove it and redo search.
|
| - revision_list.pop(next_revision_index)
|
| -
|
| - max_revision -= 1
|
| -
|
| - if self.opts.output_buildbot_annotations:
|
| - self._PrintPartialResults(results)
|
| - bisect_utils.OutputAnnotationStepClosed()
|
| - else:
|
| - # Weren't able to sync and retrieve the revision range.
|
| - results['error'] = ('An error occurred attempting to retrieve revision '
|
| - 'range: [%s..%s]' % (good_revision, bad_revision))
|
| -
|
| - return results
|
| -
|
| - def _PrintPartialResults(self, results_dict):
|
| - revision_data = results_dict['revision_data']
|
| - revision_data_sorted = sorted(revision_data.iteritems(),
|
| - key = lambda x: x[1]['sort'])
|
| - results_dict = self._GetResultsDict(revision_data, revision_data_sorted)
|
| -
|
| - self._PrintTestedCommitsTable(revision_data_sorted,
|
| - results_dict['first_working_revision'],
|
| - results_dict['last_broken_revision'],
|
| - 100, final_step=False)
|
| -
|
| - def _ConfidenceLevelStatus(self, results_dict):
|
| - if not results_dict['confidence']:
|
| - return None
|
| - confidence_status = 'Successful with %(level)s confidence%(warning)s.'
|
| - if results_dict['confidence'] >= HIGH_CONFIDENCE:
|
| - level = 'high'
|
| - else:
|
| - level = 'low'
|
| - warning = ' and warnings'
|
| - if not self.warnings:
|
| - warning = ''
|
| - return confidence_status % {'level': level, 'warning': warning}
|
| -
|
| - def _GetViewVCLinkFromDepotAndHash(self, cl, depot):
|
| - info = self.source_control.QueryRevisionInfo(cl,
|
| - self._GetDepotDirectory(depot))
|
| - if depot and DEPOT_DEPS_NAME[depot].has_key('viewvc'):
|
| - try:
|
| - # Format is "git-svn-id: svn://....@123456 <other data>"
|
| - svn_line = [i for i in info['body'].splitlines() if 'git-svn-id:' in i]
|
| - svn_revision = svn_line[0].split('@')
|
| - svn_revision = svn_revision[1].split(' ')[0]
|
| - return DEPOT_DEPS_NAME[depot]['viewvc'] + svn_revision
|
| - except IndexError:
|
| - return ''
|
| - return ''
|
| -
|
| - def _PrintRevisionInfo(self, cl, info, depot=None):
|
| - email_info = ''
|
| - if not info['email'].startswith(info['author']):
|
| - email_info = '\nEmail : %s' % info['email']
|
| - commit_link = self._GetViewVCLinkFromDepotAndHash(cl, depot)
|
| - if commit_link:
|
| - commit_info = '\nLink : %s' % commit_link
|
| - else:
|
| - commit_info = ('\nFailed to parse SVN revision from body:\n%s' %
|
| - info['body'])
|
| - print RESULTS_REVISION_INFO % {
|
| - 'subject': info['subject'],
|
| - 'author': info['author'],
|
| - 'email_info': email_info,
|
| - 'commit_info': commit_info,
|
| - 'cl': cl,
|
| - 'cl_date': info['date']
|
| - }
|
| -
|
| - def _PrintTestedCommitsHeader(self):
|
| - if self.opts.bisect_mode == BISECT_MODE_MEAN:
|
| - _PrintTableRow(
|
| - [20, 70, 14, 12, 13],
|
| - ['Depot', 'Commit SHA', 'Mean', 'Std. Error', 'State'])
|
| - elif self.opts.bisect_mode == BISECT_MODE_STD_DEV:
|
| - _PrintTableRow(
|
| - [20, 70, 14, 12, 13],
|
| - ['Depot', 'Commit SHA', 'Std. Error', 'Mean', 'State'])
|
| - elif self.opts.bisect_mode == BISECT_MODE_RETURN_CODE:
|
| - _PrintTableRow(
|
| - [20, 70, 14, 13],
|
| - ['Depot', 'Commit SHA', 'Return Code', 'State'])
|
| - else:
|
| - assert False, 'Invalid bisect_mode specified.'
|
| -
|
| - def _PrintTestedCommitsEntry(self, current_data, cl_link, state_str):
|
| - if self.opts.bisect_mode == BISECT_MODE_MEAN:
|
| - std_error = '+-%.02f' % current_data['value']['std_err']
|
| - mean = '%.02f' % current_data['value']['mean']
|
| - _PrintTableRow(
|
| - [20, 70, 12, 14, 13],
|
| - [current_data['depot'], cl_link, mean, std_error, state_str])
|
| - elif self.opts.bisect_mode == BISECT_MODE_STD_DEV:
|
| - std_error = '+-%.02f' % current_data['value']['std_err']
|
| - mean = '%.02f' % current_data['value']['mean']
|
| - _PrintTableRow(
|
| - [20, 70, 12, 14, 13],
|
| - [current_data['depot'], cl_link, std_error, mean, state_str])
|
| - elif self.opts.bisect_mode == BISECT_MODE_RETURN_CODE:
|
| - mean = '%d' % current_data['value']['mean']
|
| - _PrintTableRow(
|
| - [20, 70, 14, 13],
|
| - [current_data['depot'], cl_link, mean, state_str])
|
| -
|
| - def _PrintTestedCommitsTable(
|
| - self, revision_data_sorted, first_working_revision, last_broken_revision,
|
| - confidence, final_step=True):
|
| - print
|
| - if final_step:
|
| - print '===== TESTED COMMITS ====='
|
| - else:
|
| - print '===== PARTIAL RESULTS ====='
|
| - self._PrintTestedCommitsHeader()
|
| - state = 0
|
| - for current_id, current_data in revision_data_sorted:
|
| - if current_data['value']:
|
| - if (current_id == last_broken_revision or
|
| - current_id == first_working_revision):
|
| - # If confidence is too low, don't add this empty line since it's
|
| - # used to put focus on a suspected CL.
|
| - if confidence and final_step:
|
| - print
|
| - state += 1
|
| - if state == 2 and not final_step:
|
| - # Just want a separation between "bad" and "good" cl's.
|
| - print
|
| -
|
| - state_str = 'Bad'
|
| - if state == 1 and final_step:
|
| - state_str = 'Suspected CL'
|
| - elif state == 2:
|
| - state_str = 'Good'
|
| -
|
| - # If confidence is too low, don't bother outputting good/bad.
|
| - if not confidence:
|
| - state_str = ''
|
| - state_str = state_str.center(13, ' ')
|
| -
|
| - cl_link = self._GetViewVCLinkFromDepotAndHash(current_id,
|
| - current_data['depot'])
|
| - if not cl_link:
|
| - cl_link = current_id
|
| - self._PrintTestedCommitsEntry(current_data, cl_link, state_str)
|
| -
|
| - def _PrintReproSteps(self):
|
| - """Prints out a section of the results explaining how to run the test.
|
| -
|
| - This message includes the command used to run the test.
|
| - """
|
| - command = '$ ' + self.opts.command
|
| - if bisect_utils.IsTelemetryCommand(self.opts.command):
|
| - command += ('\nAlso consider passing --profiler=list to see available '
|
| - 'profilers.')
|
| - print REPRO_STEPS_LOCAL
|
| - if bisect_utils.IsTelemetryCommand(self.opts.command):
|
| - telemetry_command = re.sub(r'--browser=[^\s]+',
|
| - '--browser=<bot-name>',
|
| - command)
|
| - print REPRO_STEPS_TRYJOB_TELEMETRY % {'command': telemetry_command}
|
| - else:
|
| - print REPRO_STEPS_TRYJOB
|
| -
|
| - def _PrintOtherRegressions(self, other_regressions, revision_data):
|
| - """Prints a section of the results about other potential regressions."""
|
| - print
|
| - print 'Other regressions may have occurred:'
|
| - print ' %8s %70s %10s' % ('Depot'.center(8, ' '),
|
| - 'Range'.center(70, ' '), 'Confidence'.center(10, ' '))
|
| - for regression in other_regressions:
|
| - current_id, previous_id, confidence = regression
|
| - current_data = revision_data[current_id]
|
| - previous_data = revision_data[previous_id]
|
| -
|
| - current_link = self._GetViewVCLinkFromDepotAndHash(current_id,
|
| - current_data['depot'])
|
| - previous_link = self._GetViewVCLinkFromDepotAndHash(previous_id,
|
| - previous_data['depot'])
|
| -
|
| - # If we can't map it to a viewable URL, at least show the original hash.
|
| - if not current_link:
|
| - current_link = current_id
|
| - if not previous_link:
|
| - previous_link = previous_id
|
| -
|
| - print ' %8s %70s %s' % (
|
| - current_data['depot'], current_link,
|
| - ('%d%%' % confidence).center(10, ' '))
|
| - print ' %8s %70s' % (
|
| - previous_data['depot'], previous_link)
|
| - print
|
| -
|
| - def _GetResultsDict(self, revision_data, revision_data_sorted):
|
| - # Find range where it possibly broke.
|
| - first_working_revision = None
|
| - first_working_revision_index = -1
|
| - last_broken_revision = None
|
| - last_broken_revision_index = -1
|
| -
|
| - culprit_revisions = []
|
| - other_regressions = []
|
| - regression_size = 0.0
|
| - regression_std_err = 0.0
|
| - confidence = 0.0
|
| -
|
| - for i in xrange(len(revision_data_sorted)):
|
| - k, v = revision_data_sorted[i]
|
| - if v['passed'] == 1:
|
| - if not first_working_revision:
|
| - first_working_revision = k
|
| - first_working_revision_index = i
|
| -
|
| - if not v['passed']:
|
| - last_broken_revision = k
|
| - last_broken_revision_index = i
|
| -
|
| - if last_broken_revision != None and first_working_revision != None:
|
| - broken_means = []
|
| - for i in xrange(0, last_broken_revision_index + 1):
|
| - if revision_data_sorted[i][1]['value']:
|
| - broken_means.append(revision_data_sorted[i][1]['value']['values'])
|
| -
|
| - working_means = []
|
| - for i in xrange(first_working_revision_index, len(revision_data_sorted)):
|
| - if revision_data_sorted[i][1]['value']:
|
| - working_means.append(revision_data_sorted[i][1]['value']['values'])
|
| -
|
| - # Flatten the lists to calculate mean of all values.
|
| - working_mean = sum(working_means, [])
|
| - broken_mean = sum(broken_means, [])
|
| -
|
| - # Calculate the approximate size of the regression
|
| - mean_of_bad_runs = math_utils.Mean(broken_mean)
|
| - mean_of_good_runs = math_utils.Mean(working_mean)
|
| -
|
| - regression_size = 100 * math_utils.RelativeChange(mean_of_good_runs,
|
| - mean_of_bad_runs)
|
| - if math.isnan(regression_size):
|
| - regression_size = 'zero-to-nonzero'
|
| -
|
| - regression_std_err = math.fabs(math_utils.PooledStandardError(
|
| - [working_mean, broken_mean]) /
|
| - max(0.0001, min(mean_of_good_runs, mean_of_bad_runs))) * 100.0
|
| -
|
| - # Give a "confidence" in the bisect. At the moment we use how distinct the
|
| - # values are before and after the last broken revision, and how noisy the
|
| - # overall graph is.
|
| - confidence = ConfidenceScore(working_means, broken_means)
|
| -
|
| - culprit_revisions = []
|
| -
|
| - cwd = os.getcwd()
|
| - self.ChangeToDepotWorkingDirectory(
|
| - revision_data[last_broken_revision]['depot'])
|
| -
|
| - if revision_data[last_broken_revision]['depot'] == 'cros':
|
| - # Want to get a list of all the commits and what depots they belong
|
| - # to so that we can grab info about each.
|
| - cmd = ['repo', 'forall', '-c',
|
| - 'pwd ; git log --pretty=oneline --before=%d --after=%d' % (
|
| - last_broken_revision, first_working_revision + 1)]
|
| - output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd)
|
| -
|
| - changes = []
|
| - assert not return_code, ('An error occurred while running '
|
| - '"%s"' % ' '.join(cmd))
|
| - last_depot = None
|
| - cwd = os.getcwd()
|
| - for l in output.split('\n'):
|
| - if l:
|
| - # Output will be in form:
|
| - # /path_to_depot
|
| - # /path_to_other_depot
|
| - # <SHA1>
|
| - # /path_again
|
| - # <SHA1>
|
| - # etc.
|
| - if l[0] == '/':
|
| - last_depot = l
|
| - else:
|
| - contents = l.split(' ')
|
| - if len(contents) > 1:
|
| - changes.append([last_depot, contents[0]])
|
| - for c in changes:
|
| - os.chdir(c[0])
|
| - info = self.source_control.QueryRevisionInfo(c[1])
|
| - culprit_revisions.append((c[1], info, None))
|
| - else:
|
| - for i in xrange(last_broken_revision_index, len(revision_data_sorted)):
|
| - k, v = revision_data_sorted[i]
|
| - if k == first_working_revision:
|
| - break
|
| - self.ChangeToDepotWorkingDirectory(v['depot'])
|
| - info = self.source_control.QueryRevisionInfo(k)
|
| - culprit_revisions.append((k, info, v['depot']))
|
| - os.chdir(cwd)
|
| -
|
| - # Check for any other possible regression ranges.
|
| - other_regressions = _FindOtherRegressions(
|
| - revision_data_sorted, mean_of_bad_runs > mean_of_good_runs)
|
| -
|
| - return {
|
| - 'first_working_revision': first_working_revision,
|
| - 'last_broken_revision': last_broken_revision,
|
| - 'culprit_revisions': culprit_revisions,
|
| - 'other_regressions': other_regressions,
|
| - 'regression_size': regression_size,
|
| - 'regression_std_err': regression_std_err,
|
| - 'confidence': confidence,
|
| - }
|
| -
|
| - def _CheckForWarnings(self, results_dict):
|
| - if len(results_dict['culprit_revisions']) > 1:
|
| - self.warnings.append('Due to build errors, regression range could '
|
| - 'not be narrowed down to a single commit.')
|
| - if self.opts.repeat_test_count == 1:
|
| - self.warnings.append('Tests were only set to run once. This may '
|
| - 'be insufficient to get meaningful results.')
|
| - if 0 < results_dict['confidence'] < HIGH_CONFIDENCE:
|
| - self.warnings.append('Confidence is not high. Try bisecting again '
|
| - 'with increased repeat_count, larger range, or '
|
| - 'on another metric.')
|
| - if not results_dict['confidence']:
|
| - self.warnings.append('Confidence score is 0%. Try bisecting again on '
|
| - 'another platform or another metric.')
|
| -
|
| - def FormatAndPrintResults(self, bisect_results):
|
| - """Prints the results from a bisection run in a readable format.
|
| -
|
| - Args:
|
| - bisect_results: The results from a bisection test run.
|
| - """
|
| - revision_data = bisect_results['revision_data']
|
| - revision_data_sorted = sorted(revision_data.iteritems(),
|
| - key = lambda x: x[1]['sort'])
|
| - results_dict = self._GetResultsDict(revision_data, revision_data_sorted)
|
| -
|
| - self._CheckForWarnings(results_dict)
|
| -
|
| - if self.opts.output_buildbot_annotations:
|
| - bisect_utils.OutputAnnotationStepStart('Build Status Per Revision')
|
| -
|
| - print
|
| - print 'Full results of bisection:'
|
| - for current_id, current_data in revision_data_sorted:
|
| - build_status = current_data['passed']
|
| -
|
| - if type(build_status) is bool:
|
| - if build_status:
|
| - build_status = 'Good'
|
| - else:
|
| - build_status = 'Bad'
|
| -
|
| - print ' %20s %40s %s' % (current_data['depot'],
|
| - current_id, build_status)
|
| - print
|
| -
|
| - if self.opts.output_buildbot_annotations:
|
| - bisect_utils.OutputAnnotationStepClosed()
|
| - # The perf dashboard scrapes the "results" step in order to comment on
|
| - # bugs. If you change this, please update the perf dashboard as well.
|
| - bisect_utils.OutputAnnotationStepStart('Results')
|
| -
|
| - self._PrintBanner(results_dict)
|
| - self._PrintWarnings()
|
| -
|
| - if results_dict['culprit_revisions'] and results_dict['confidence']:
|
| - for culprit in results_dict['culprit_revisions']:
|
| - cl, info, depot = culprit
|
| - self._PrintRevisionInfo(cl, info, depot)
|
| - if results_dict['other_regressions']:
|
| - self._PrintOtherRegressions(results_dict['other_regressions'],
|
| - revision_data)
|
| - self._PrintTestedCommitsTable(revision_data_sorted,
|
| - results_dict['first_working_revision'],
|
| - results_dict['last_broken_revision'],
|
| - results_dict['confidence'])
|
| - _PrintStepTime(revision_data_sorted)
|
| - self._PrintReproSteps()
|
| - _PrintThankYou()
|
| - if self.opts.output_buildbot_annotations:
|
| - bisect_utils.OutputAnnotationStepClosed()
|
| -
|
| - def _PrintBanner(self, results_dict):
|
| - if self._IsBisectModeReturnCode():
|
| - metrics = 'N/A'
|
| - change = 'Yes'
|
| - else:
|
| - metrics = '/'.join(self.opts.metric)
|
| - change = '%.02f%% (+/-%.02f%%)' % (
|
| - results_dict['regression_size'], results_dict['regression_std_err'])
|
| -
|
| - if results_dict['culprit_revisions'] and results_dict['confidence']:
|
| - status = self._ConfidenceLevelStatus(results_dict)
|
| - else:
|
| - status = 'Failure, could not reproduce.'
|
| - change = 'Bisect could not reproduce a change.'
|
| -
|
| - print RESULTS_BANNER % {
|
| - 'status': status,
|
| - 'command': self.opts.command,
|
| - 'metrics': metrics,
|
| - 'change': change,
|
| - 'confidence': results_dict['confidence'],
|
| - }
|
| -
|
| - def _PrintWarnings(self):
|
| - """Prints a list of warning strings if there are any."""
|
| - if not self.warnings:
|
| - return
|
| - print
|
| - print 'WARNINGS:'
|
| - for w in set(self.warnings):
|
| - print ' ! %s' % w
|
| -
|
| -
|
| -def _IsPlatformSupported():
|
| - """Checks that this platform and build system are supported.
|
| -
|
| - Args:
|
| - opts: The options parsed from the command line.
|
| -
|
| - Returns:
|
| - True if the platform and build system are supported.
|
| - """
|
| - # Haven't tested the script out on any other platforms yet.
|
| - supported = ['posix', 'nt']
|
| - return os.name in supported
|
| -
|
| -
|
| -def RmTreeAndMkDir(path_to_dir, skip_makedir=False):
|
| - """Removes the directory tree specified, and then creates an empty
|
| - directory in the same location (if not specified to skip).
|
| -
|
| - Args:
|
| - path_to_dir: Path to the directory tree.
|
| - skip_makedir: Whether to skip creating empty directory, default is False.
|
| -
|
| - Returns:
|
| - True if successful, False if an error occurred.
|
| - """
|
| - try:
|
| - if os.path.exists(path_to_dir):
|
| - shutil.rmtree(path_to_dir)
|
| - except OSError, e:
|
| - if e.errno != errno.ENOENT:
|
| - return False
|
| -
|
| - if not skip_makedir:
|
| - return MaybeMakeDirectory(path_to_dir)
|
| -
|
| - return True
|
| -
|
| -
|
| -def RemoveBuildFiles(build_type):
|
| - """Removes build files from previous runs."""
|
| - if RmTreeAndMkDir(os.path.join('out', build_type)):
|
| - if RmTreeAndMkDir(os.path.join('build', build_type)):
|
| - return True
|
| - return False
|
| -
|
| -
|
| -class BisectOptions(object):
|
| - """Options to be used when running bisection."""
|
| - def __init__(self):
|
| - super(BisectOptions, self).__init__()
|
| -
|
| - self.target_platform = 'chromium'
|
| - self.build_preference = None
|
| - self.good_revision = None
|
| - self.bad_revision = None
|
| - self.use_goma = None
|
| - self.goma_dir = None
|
| - self.cros_board = None
|
| - self.cros_remote_ip = None
|
| - self.repeat_test_count = 20
|
| - self.truncate_percent = 25
|
| - self.max_time_minutes = 20
|
| - self.metric = None
|
| - self.command = None
|
| - self.output_buildbot_annotations = None
|
| - self.no_custom_deps = False
|
| - self.working_directory = None
|
| - self.extra_src = None
|
| - self.debug_ignore_build = None
|
| - self.debug_ignore_sync = None
|
| - self.debug_ignore_perf_test = None
|
| - self.gs_bucket = None
|
| - self.target_arch = 'ia32'
|
| - self.target_build_type = 'Release'
|
| - self.builder_host = None
|
| - self.builder_port = None
|
| - self.bisect_mode = BISECT_MODE_MEAN
|
| -
|
| - @staticmethod
|
| - def _CreateCommandLineParser():
|
| - """Creates a parser with bisect options.
|
| -
|
| - Returns:
|
| - An instance of optparse.OptionParser.
|
| - """
|
| - usage = ('%prog [options] [-- chromium-options]\n'
|
| - 'Perform binary search on revision history to find a minimal '
|
| - 'range of revisions where a performance metric regressed.\n')
|
| -
|
| - parser = optparse.OptionParser(usage=usage)
|
| -
|
| - group = optparse.OptionGroup(parser, 'Bisect options')
|
| - group.add_option('-c', '--command',
|
| - type='str',
|
| - help='A command to execute your performance test at' +
|
| - ' each point in the bisection.')
|
| - group.add_option('-b', '--bad_revision',
|
| - type='str',
|
| - help='A bad revision to start bisection. ' +
|
| - 'Must be later than good revision. May be either a git' +
|
| - ' or svn revision.')
|
| - group.add_option('-g', '--good_revision',
|
| - type='str',
|
| - help='A revision to start bisection where performance' +
|
| - ' test is known to pass. Must be earlier than the ' +
|
| - 'bad revision. May be either a git or svn revision.')
|
| - group.add_option('-m', '--metric',
|
| - type='str',
|
| - help='The desired metric to bisect on. For example ' +
|
| - '"vm_rss_final_b/vm_rss_f_b"')
|
| - group.add_option('-r', '--repeat_test_count',
|
| - type='int',
|
| - default=20,
|
| - help='The number of times to repeat the performance '
|
| - 'test. Values will be clamped to range [1, 100]. '
|
| - 'Default value is 20.')
|
| - group.add_option('--max_time_minutes',
|
| - type='int',
|
| - default=20,
|
| - help='The maximum time (in minutes) to take running the '
|
| - 'performance tests. The script will run the performance '
|
| - 'tests according to --repeat_test_count, so long as it '
|
| - 'doesn\'t exceed --max_time_minutes. Values will be '
|
| - 'clamped to range [1, 60].'
|
| - 'Default value is 20.')
|
| - group.add_option('-t', '--truncate_percent',
|
| - type='int',
|
| - default=25,
|
| - help='The highest/lowest % are discarded to form a '
|
| - 'truncated mean. Values will be clamped to range [0, '
|
| - '25]. Default value is 25 (highest/lowest 25% will be '
|
| - 'discarded).')
|
| - group.add_option('--bisect_mode',
|
| - type='choice',
|
| - choices=[BISECT_MODE_MEAN, BISECT_MODE_STD_DEV,
|
| - BISECT_MODE_RETURN_CODE],
|
| - default=BISECT_MODE_MEAN,
|
| - help='The bisect mode. Choices are to bisect on the '
|
| - 'difference in mean, std_dev, or return_code.')
|
| - parser.add_option_group(group)
|
| -
|
| - group = optparse.OptionGroup(parser, 'Build options')
|
| - group.add_option('-w', '--working_directory',
|
| - type='str',
|
| - help='Path to the working directory where the script '
|
| - 'will do an initial checkout of the chromium depot. The '
|
| - 'files will be placed in a subdirectory "bisect" under '
|
| - 'working_directory and that will be used to perform the '
|
| - 'bisection. This parameter is optional, if it is not '
|
| - 'supplied, the script will work from the current depot.')
|
| - group.add_option('--build_preference',
|
| - type='choice',
|
| - choices=['msvs', 'ninja', 'make'],
|
| - help='The preferred build system to use. On linux/mac '
|
| - 'the options are make/ninja. On Windows, the options '
|
| - 'are msvs/ninja.')
|
| - group.add_option('--target_platform',
|
| - type='choice',
|
| - choices=['chromium', 'cros', 'android', 'android-chrome'],
|
| - default='chromium',
|
| - help='The target platform. Choices are "chromium" '
|
| - '(current platform), "cros", or "android". If you '
|
| - 'specify something other than "chromium", you must be '
|
| - 'properly set up to build that platform.')
|
| - group.add_option('--no_custom_deps',
|
| - dest='no_custom_deps',
|
| - action='store_true',
|
| - default=False,
|
| - help='Run the script with custom_deps or not.')
|
| - group.add_option('--extra_src',
|
| - type='str',
|
| - help='Path to a script which can be used to modify '
|
| - 'the bisect script\'s behavior.')
|
| - group.add_option('--cros_board',
|
| - type='str',
|
| - help='The cros board type to build.')
|
| - group.add_option('--cros_remote_ip',
|
| - type='str',
|
| - help='The remote machine to image to.')
|
| - group.add_option('--use_goma',
|
| - action='store_true',
|
| - help='Add a bunch of extra threads for goma, and enable '
|
| - 'goma')
|
| - group.add_option('--goma_dir',
|
| - help='Path to goma tools (or system default if not '
|
| - 'specified).')
|
| - group.add_option('--output_buildbot_annotations',
|
| - action='store_true',
|
| - help='Add extra annotation output for buildbot.')
|
| - group.add_option('--gs_bucket',
|
| - default='',
|
| - dest='gs_bucket',
|
| - type='str',
|
| - help=('Name of Google Storage bucket to upload or '
|
| - 'download build. e.g., chrome-perf'))
|
| - group.add_option('--target_arch',
|
| - type='choice',
|
| - choices=['ia32', 'x64', 'arm'],
|
| - default='ia32',
|
| - dest='target_arch',
|
| - help=('The target build architecture. Choices are "ia32" '
|
| - '(default), "x64" or "arm".'))
|
| - group.add_option('--target_build_type',
|
| - type='choice',
|
| - choices=['Release', 'Debug'],
|
| - default='Release',
|
| - help='The target build type. Choices are "Release" '
|
| - '(default), or "Debug".')
|
| - group.add_option('--builder_host',
|
| - dest='builder_host',
|
| - type='str',
|
| - help=('Host address of server to produce build by posting'
|
| - ' try job request.'))
|
| - group.add_option('--builder_port',
|
| - dest='builder_port',
|
| - type='int',
|
| - help=('HTTP port of the server to produce build by posting'
|
| - ' try job request.'))
|
| - parser.add_option_group(group)
|
| -
|
| - group = optparse.OptionGroup(parser, 'Debug options')
|
| - group.add_option('--debug_ignore_build',
|
| - action='store_true',
|
| - help='DEBUG: Don\'t perform builds.')
|
| - group.add_option('--debug_ignore_sync',
|
| - action='store_true',
|
| - help='DEBUG: Don\'t perform syncs.')
|
| - group.add_option('--debug_ignore_perf_test',
|
| - action='store_true',
|
| - help='DEBUG: Don\'t perform performance tests.')
|
| - parser.add_option_group(group)
|
| - return parser
|
| -
|
| - def ParseCommandLine(self):
|
| - """Parses the command line for bisect options."""
|
| - parser = self._CreateCommandLineParser()
|
| - opts, _ = parser.parse_args()
|
| -
|
| - try:
|
| - if not opts.command:
|
| - raise RuntimeError('missing required parameter: --command')
|
| -
|
| - if not opts.good_revision:
|
| - raise RuntimeError('missing required parameter: --good_revision')
|
| -
|
| - if not opts.bad_revision:
|
| - raise RuntimeError('missing required parameter: --bad_revision')
|
| -
|
| - if not opts.metric and opts.bisect_mode != BISECT_MODE_RETURN_CODE:
|
| - raise RuntimeError('missing required parameter: --metric')
|
| -
|
| - if opts.gs_bucket:
|
| - if not cloud_storage.List(opts.gs_bucket):
|
| - raise RuntimeError('Invalid Google Storage: gs://%s' % opts.gs_bucket)
|
| - if not opts.builder_host:
|
| - raise RuntimeError('Must specify try server host name using '
|
| - '--builder_host when gs_bucket is used.')
|
| - if not opts.builder_port:
|
| - raise RuntimeError('Must specify try server port number using '
|
| - '--builder_port when gs_bucket is used.')
|
| - if opts.target_platform == 'cros':
|
| - # Run sudo up front to make sure credentials are cached for later.
|
| - print 'Sudo is required to build cros:'
|
| - print
|
| - bisect_utils.RunProcess(['sudo', 'true'])
|
| -
|
| - if not opts.cros_board:
|
| - raise RuntimeError('missing required parameter: --cros_board')
|
| -
|
| - if not opts.cros_remote_ip:
|
| - raise RuntimeError('missing required parameter: --cros_remote_ip')
|
| -
|
| - if not opts.working_directory:
|
| - raise RuntimeError('missing required parameter: --working_directory')
|
| -
|
| - if opts.bisect_mode != BISECT_MODE_RETURN_CODE:
|
| - metric_values = opts.metric.split('/')
|
| - if len(metric_values) != 2:
|
| - raise RuntimeError('Invalid metric specified: [%s]' % opts.metric)
|
| - opts.metric = metric_values
|
| -
|
| - opts.repeat_test_count = min(max(opts.repeat_test_count, 1), 100)
|
| - opts.max_time_minutes = min(max(opts.max_time_minutes, 1), 60)
|
| - opts.truncate_percent = min(max(opts.truncate_percent, 0), 25)
|
| - opts.truncate_percent = opts.truncate_percent / 100.0
|
| -
|
| - for k, v in opts.__dict__.iteritems():
|
| - assert hasattr(self, k), 'Invalid %s attribute in BisectOptions.' % k
|
| - setattr(self, k, v)
|
| - except RuntimeError, e:
|
| - output_string = StringIO.StringIO()
|
| - parser.print_help(file=output_string)
|
| - error_message = '%s\n\n%s' % (e.message, output_string.getvalue())
|
| - output_string.close()
|
| - raise RuntimeError(error_message)
|
| -
|
| - @staticmethod
|
| - def FromDict(values):
|
| - """Creates an instance of BisectOptions from a dictionary.
|
| -
|
| - Args:
|
| - values: a dict containing options to set.
|
| -
|
| - Returns:
|
| - An instance of BisectOptions.
|
| - """
|
| - opts = BisectOptions()
|
| - for k, v in values.iteritems():
|
| - assert hasattr(opts, k), 'Invalid %s attribute in BisectOptions.' % k
|
| - setattr(opts, k, v)
|
| -
|
| - if opts.metric and opts.bisect_mode != BISECT_MODE_RETURN_CODE:
|
| - metric_values = opts.metric.split('/')
|
| - if len(metric_values) != 2:
|
| - raise RuntimeError('Invalid metric specified: [%s]' % opts.metric)
|
| - opts.metric = metric_values
|
| -
|
| - opts.repeat_test_count = min(max(opts.repeat_test_count, 1), 100)
|
| - opts.max_time_minutes = min(max(opts.max_time_minutes, 1), 60)
|
| - opts.truncate_percent = min(max(opts.truncate_percent, 0), 25)
|
| - opts.truncate_percent = opts.truncate_percent / 100.0
|
| -
|
| - return opts
|
| -
|
| -
|
| -def main():
|
| -
|
| - try:
|
| - opts = BisectOptions()
|
| - opts.ParseCommandLine()
|
| -
|
| - if opts.extra_src:
|
| - extra_src = bisect_utils.LoadExtraSrc(opts.extra_src)
|
| - if not extra_src:
|
| - raise RuntimeError('Invalid or missing --extra_src.')
|
| - _AddAdditionalDepotInfo(extra_src.GetAdditionalDepotInfo())
|
| -
|
| - if opts.working_directory:
|
| - custom_deps = bisect_utils.DEFAULT_GCLIENT_CUSTOM_DEPS
|
| - if opts.no_custom_deps:
|
| - custom_deps = None
|
| - bisect_utils.CreateBisectDirectoryAndSetupDepot(opts, custom_deps)
|
| -
|
| - os.chdir(os.path.join(os.getcwd(), 'src'))
|
| -
|
| - if not RemoveBuildFiles(opts.target_build_type):
|
| - raise RuntimeError('Something went wrong removing the build files.')
|
| -
|
| - if not _IsPlatformSupported():
|
| - raise RuntimeError('Sorry, this platform isn\'t supported yet.')
|
| -
|
| - # Check what source control method is being used, and create a
|
| - # SourceControl object if possible.
|
| - source_control = source_control_module.DetermineAndCreateSourceControl(opts)
|
| -
|
| - if not source_control:
|
| - raise RuntimeError(
|
| - 'Sorry, only the git workflow is supported at the moment.')
|
| -
|
| - # gClient sync seems to fail if you're not in master branch.
|
| - if (not source_control.IsInProperBranch() and
|
| - not opts.debug_ignore_sync and
|
| - not opts.working_directory):
|
| - raise RuntimeError('You must switch to master branch to run bisection.')
|
| - bisect_test = BisectPerformanceMetrics(source_control, opts)
|
| - try:
|
| - bisect_results = bisect_test.Run(opts.command,
|
| - opts.bad_revision,
|
| - opts.good_revision,
|
| - opts.metric)
|
| - if bisect_results['error']:
|
| - raise RuntimeError(bisect_results['error'])
|
| - bisect_test.FormatAndPrintResults(bisect_results)
|
| - return 0
|
| - finally:
|
| - bisect_test.PerformCleanup()
|
| - except RuntimeError, e:
|
| - if opts.output_buildbot_annotations:
|
| - # The perf dashboard scrapes the "results" step in order to comment on
|
| - # bugs. If you change this, please update the perf dashboard as well.
|
| - bisect_utils.OutputAnnotationStepStart('Results')
|
| - print 'Error: %s' % e.message
|
| - if opts.output_buildbot_annotations:
|
| - bisect_utils.OutputAnnotationStepClosed()
|
| - return 1
|
| -
|
| -
|
| -if __name__ == '__main__':
|
| - sys.exit(main())
|
|
|
Chromium Code Reviews
Issue 564663002:
Move bisect-perf-regression.py into auto_bisect directory. (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master