| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Performance Test Bisect Tool | 6 """Performance Test Bisect Tool |
| 7 | 7 |
| 8 This script bisects a series of changelists using binary search. It starts at | 8 This script bisects a series of changelists using binary search. It starts at |
| 9 a bad revision where a performance metric has regressed, and asks for a last | 9 a bad revision where a performance metric has regressed, and asks for a last |
| 10 known-good revision. It will then binary search across this revision range by | 10 known-good revision. It will then binary search across this revision range by |
| 11 syncing, building, and running a performance test. If the change is | 11 syncing, building, and running a performance test. If the change is |
| 12 suspected to occur as a result of WebKit/V8 changes, the script will | 12 suspected to occur as a result of WebKit/V8 changes, the script will |
| 13 further bisect changes to those depots and attempt to narrow down the revision | 13 further bisect changes to those depots and attempt to narrow down the revision |
| 14 range. | 14 range. |
| 15 | 15 |
| 16 | 16 |
| 17 An example usage (using svn cl's): | 17 An example usage (using svn cl's): |
| 18 | 18 |
| 19 ./tools/bisect-perf-regression.py -c\ | 19 ./tools/bisect-perf-regression.py -c\ |
| 20 "out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\ | 20 "out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\ |
| 21 -g 168222 -b 168232 -m shutdown/simple-user-quit | 21 -g 168222 -b 168232 -m shutdown/simple-user-quit |
| 22 | 22 |
| 23 Be aware that if you're using the git workflow and specify an svn revision, | 23 Be aware that if you're using the git workflow and specify an svn revision, |
| 24 the script will attempt to find the git SHA1 where svn changes up to that | 24 the script will attempt to find the git SHA1 where svn changes up to that |
| 25 revision were merged in. | 25 revision were merged in. |
| 26 | 26 |
| 27 | 27 |
| 28 An example usage (using git hashes): | 28 An example usage (using git hashes): |
| 29 | 29 |
| 30 ./tools/bisect-perf-regression.py -c\ | 30 ./tools/bisect-perf-regression.py -c\ |
| 31 "out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\ | 31 "out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\ |
| 32 -g 1f6e67861535121c5c819c16a666f2436c207e7b\ | 32 -g 1f6e67861535121c5c819c16a666f2436c207e7b\ |
| 33 -b b732f23b4f81c382db0b23b9035f3dadc7d925bb\ | 33 -b b732f23b4f81c382db0b23b9035f3dadc7d925bb\ |
| 34 -m shutdown/simple-user-quit | 34 -m shutdown/simple-user-quit |
| 35 """ | 35 """ |
| 36 | 36 |
| 37 import copy | 37 import copy |
| 38 import datetime | 38 import datetime |
| 39 import errno | 39 import errno |
| 40 import hashlib | 40 import hashlib |
| 41 import math | 41 import math |
| 42 import optparse | 42 import optparse |
| 43 import os | 43 import os |
| 44 import re | 44 import re |
| 45 import shlex | 45 import shlex |
| 46 import shutil | 46 import shutil |
| 47 import StringIO | 47 import StringIO |
| 48 import sys | 48 import sys |
| 49 import time | 49 import time |
| 50 import zipfile | 50 import zipfile |
| 51 | 51 |
| 52 sys.path.append(os.path.join(os.path.dirname(__file__), 'telemetry')) | 52 sys.path.append(os.path.join(os.path.dirname(__file__), 'telemetry')) |
| 53 | 53 |
| 54 from auto_bisect import bisect_utils | 54 from auto_bisect import bisect_utils |
| 55 from auto_bisect import math_utils | 55 from auto_bisect import math_utils |
| 56 from auto_bisect import post_perf_builder_job as bisect_builder | 56 from auto_bisect import post_perf_builder_job as bisect_builder |
| 57 from auto_bisect import source_control as source_control_module | 57 from auto_bisect import source_control as source_control_module |
| 58 from auto_bisect import ttest | 58 from auto_bisect import ttest |
| 59 from telemetry.util import cloud_storage | 59 from telemetry.util import cloud_storage |
| 60 | 60 |
| 61 # The additional repositories that might need to be bisected. | 61 # Below is the map of "depot" names to information about each depot. Each depot |
| 62 # If the repository has any dependant repositories (such as skia/src needs | 62 # is a repository, and in the process of bisecting, revision ranges in these |
| 63 # skia/include and skia/gyp to be updated), specify them in the 'depends' | 63 # repositories may also be bisected. |
| 64 # so that they're synced appropriately. | 64 # |
| 65 # Format is: | 65 # Each depot information dictionary may contain: |
| 66 # src: path to the working directory. | 66 # src: Path to the working directory. |
| 67 # recurse: True if this repositry will get bisected. | 67 # recurse: True if this repository will get bisected. |
| 68 # depends: A list of other repositories that are actually part of the same | 68 # depends: A list of other repositories that are actually part of the same |
| 69 # repository in svn. | 69 # repository in svn. If the repository has any dependent repositories |
| 70 # svn: Needed for git workflow to resolve hashes to svn revisions. | 70 # (e.g. skia/src needs skia/include and skia/gyp to be updated), then |
| 71 # from: Parent depot that must be bisected before this is bisected. | 71 # they are specified here. |
| 72 # deps_var: Key name in vars varible in DEPS file that has revision information. | 72 # svn: URL of SVN repository. Needed for git workflow to resolve hashes to |
| 73 # SVN revisions. |
| 74 # from: Parent depot that must be bisected before this is bisected. |
| 75 # deps_var: Key name in vars variable in DEPS file that has revision |
| 76 # information. |
| 73 DEPOT_DEPS_NAME = { | 77 DEPOT_DEPS_NAME = { |
| 74 'chromium' : { | 78 'chromium': { |
| 75 "src" : "src", | 79 'src': 'src', |
| 76 "recurse" : True, | 80 'recurse': True, |
| 77 "depends" : None, | 81 'depends': None, |
| 78 "from" : ['cros', 'android-chrome'], | 82 'from': ['cros', 'android-chrome'], |
| 79 'viewvc': 'http://src.chromium.org/viewvc/chrome?view=revision&revision=', | 83 'viewvc': |
| 80 'deps_var': 'chromium_rev' | 84 'http://src.chromium.org/viewvc/chrome?view=revision&revision=', |
| 81 }, | 85 'deps_var': 'chromium_rev' |
| 82 'webkit' : { | 86 }, |
| 83 "src" : "src/third_party/WebKit", | 87 'webkit': { |
| 84 "recurse" : True, | 88 'src': 'src/third_party/WebKit', |
| 85 "depends" : None, | 89 'recurse': True, |
| 86 "from" : ['chromium'], | 90 'depends': None, |
| 87 'viewvc': 'http://src.chromium.org/viewvc/blink?view=revision&revision=', | 91 'from': ['chromium'], |
| 88 'deps_var': 'webkit_revision' | 92 'viewvc': |
| 89 }, | 93 'http://src.chromium.org/viewvc/blink?view=revision&revision=', |
| 90 'angle' : { | 94 'deps_var': 'webkit_revision' |
| 91 "src" : "src/third_party/angle", | 95 }, |
| 92 "src_old" : "src/third_party/angle_dx11", | 96 'angle': { |
| 93 "recurse" : True, | 97 'src': 'src/third_party/angle', |
| 94 "depends" : None, | 98 'src_old': 'src/third_party/angle_dx11', |
| 95 "from" : ['chromium'], | 99 'recurse': True, |
| 96 "platform": 'nt', | 100 'depends': None, |
| 97 'deps_var': 'angle_revision' | 101 'from': ['chromium'], |
| 98 }, | 102 'platform': 'nt', |
| 99 'v8' : { | 103 'deps_var': 'angle_revision' |
| 100 "src" : "src/v8", | 104 }, |
| 101 "recurse" : True, | 105 'v8': { |
| 102 "depends" : None, | 106 'src': 'src/v8', |
| 103 "from" : ['chromium'], | 107 'recurse': True, |
| 104 "custom_deps": bisect_utils.GCLIENT_CUSTOM_DEPS_V8, | 108 'depends': None, |
| 105 'viewvc': 'https://code.google.com/p/v8/source/detail?r=', | 109 'from': ['chromium'], |
| 106 'deps_var': 'v8_revision' | 110 'custom_deps': bisect_utils.GCLIENT_CUSTOM_DEPS_V8, |
| 107 }, | 111 'viewvc': 'https://code.google.com/p/v8/source/detail?r=', |
| 108 'v8_bleeding_edge' : { | 112 'deps_var': 'v8_revision' |
| 109 "src" : "src/v8_bleeding_edge", | 113 }, |
| 110 "recurse" : True, | 114 'v8_bleeding_edge': { |
| 111 "depends" : None, | 115 'src': 'src/v8_bleeding_edge', |
| 112 "svn": "https://v8.googlecode.com/svn/branches/bleeding_edge", | 116 'recurse': True, |
| 113 "from" : ['v8'], | 117 'depends': None, |
| 114 'viewvc': 'https://code.google.com/p/v8/source/detail?r=', | 118 'svn': 'https://v8.googlecode.com/svn/branches/bleeding_edge', |
| 115 'deps_var': 'v8_revision' | 119 'from': ['v8'], |
| 116 }, | 120 'viewvc': 'https://code.google.com/p/v8/source/detail?r=', |
| 117 'skia/src' : { | 121 'deps_var': 'v8_revision' |
| 118 "src" : "src/third_party/skia/src", | 122 }, |
| 119 "recurse" : True, | 123 'skia/src': { |
| 120 "svn" : "http://skia.googlecode.com/svn/trunk/src", | 124 'src': 'src/third_party/skia/src', |
| 121 "depends" : ['skia/include', 'skia/gyp'], | 125 'recurse': True, |
| 122 "from" : ['chromium'], | 126 'svn': 'http://skia.googlecode.com/svn/trunk/src', |
| 123 'viewvc': 'https://code.google.com/p/skia/source/detail?r=', | 127 'depends': ['skia/include', 'skia/gyp'], |
| 124 'deps_var': 'skia_revision' | 128 'from': ['chromium'], |
| 125 }, | 129 'viewvc': 'https://code.google.com/p/skia/source/detail?r=', |
| 126 'skia/include' : { | 130 'deps_var': 'skia_revision' |
| 127 "src" : "src/third_party/skia/include", | 131 }, |
| 128 "recurse" : False, | 132 'skia/include': { |
| 129 "svn" : "http://skia.googlecode.com/svn/trunk/include", | 133 'src': 'src/third_party/skia/include', |
| 130 "depends" : None, | 134 'recurse': False, |
| 131 "from" : ['chromium'], | 135 'svn': 'http://skia.googlecode.com/svn/trunk/include', |
| 132 'viewvc': 'https://code.google.com/p/skia/source/detail?r=', | 136 'depends': None, |
| 133 'deps_var': 'None' | 137 'from': ['chromium'], |
| 134 }, | 138 'viewvc': 'https://code.google.com/p/skia/source/detail?r=', |
| 135 'skia/gyp' : { | 139 'deps_var': 'None' |
| 136 "src" : "src/third_party/skia/gyp", | 140 }, |
| 137 "recurse" : False, | 141 'skia/gyp': { |
| 138 "svn" : "http://skia.googlecode.com/svn/trunk/gyp", | 142 'src': 'src/third_party/skia/gyp', |
| 139 "depends" : None, | 143 'recurse': False, |
| 140 "from" : ['chromium'], | 144 'svn': 'http://skia.googlecode.com/svn/trunk/gyp', |
| 141 'viewvc': 'https://code.google.com/p/skia/source/detail?r=', | 145 'depends': None, |
| 142 'deps_var': 'None' | 146 'from': ['chromium'], |
| 143 }, | 147 'viewvc': 'https://code.google.com/p/skia/source/detail?r=', |
| 148 'deps_var': 'None' |
| 149 } |
| 144 } | 150 } |
| 145 | 151 |
| 146 DEPOT_NAMES = DEPOT_DEPS_NAME.keys() | 152 DEPOT_NAMES = DEPOT_DEPS_NAME.keys() |
| 147 | 153 |
| 148 CROS_SDK_PATH = os.path.join('..', 'cros', 'chromite', 'bin', 'cros_sdk') | 154 CROS_SDK_PATH = os.path.join('..', 'cros', 'chromite', 'bin', 'cros_sdk') |
| 149 CROS_CHROMEOS_PATTERN = 'chromeos-base/chromeos-chrome' | 155 CROS_CHROMEOS_PATTERN = 'chromeos-base/chromeos-chrome' |
| 150 CROS_TEST_KEY_PATH = os.path.join('..', 'cros', 'chromite', 'ssh_keys', | 156 CROS_TEST_KEY_PATH = os.path.join('..', 'cros', 'chromite', 'ssh_keys', |
| 151 'testing_rsa') | 157 'testing_rsa') |
| 152 CROS_SCRIPT_KEY_PATH = os.path.join('..', 'cros', 'src', 'scripts', | 158 CROS_SCRIPT_KEY_PATH = os.path.join('..', 'cros', 'src', 'scripts', |
| 153 'mod_for_test_scripts', 'ssh_keys', | 159 'mod_for_test_scripts', 'ssh_keys', |
| 154 'testing_rsa') | 160 'testing_rsa') |
| 155 | 161 |
| 162 # Possible return values from BisectPerformanceMetrics.SyncBuildAndRunRevision. |
| 156 BUILD_RESULT_SUCCEED = 0 | 163 BUILD_RESULT_SUCCEED = 0 |
| 157 BUILD_RESULT_FAIL = 1 | 164 BUILD_RESULT_FAIL = 1 |
| 158 BUILD_RESULT_SKIPPED = 2 | 165 BUILD_RESULT_SKIPPED = 2 |
| 159 | 166 |
| 160 # Maximum time in seconds to wait after posting build request to tryserver. | 167 # Maximum time in seconds to wait after posting build request to tryserver. |
| 161 # TODO: Change these values based on the actual time taken by buildbots on | 168 # TODO: Change these values based on the actual time taken by buildbots on |
| 162 # the tryserver. | 169 # the tryserver. |
| 163 MAX_MAC_BUILD_TIME = 14400 | 170 MAX_MAC_BUILD_TIME = 14400 |
| 164 MAX_WIN_BUILD_TIME = 14400 | 171 MAX_WIN_BUILD_TIME = 14400 |
| 165 MAX_LINUX_BUILD_TIME = 14400 | 172 MAX_LINUX_BUILD_TIME = 14400 |
| (...skipping 12 matching lines...) Expand all Loading... |
| 178 +%(deps_sha)s | 185 +%(deps_sha)s |
| 179 """ | 186 """ |
| 180 | 187 |
| 181 # The possible values of the --bisect_mode flag, which determines what to | 188 # The possible values of the --bisect_mode flag, which determines what to |
| 182 # use when classifying a revision as "good" or "bad". | 189 # use when classifying a revision as "good" or "bad". |
| 183 BISECT_MODE_MEAN = 'mean' | 190 BISECT_MODE_MEAN = 'mean' |
| 184 BISECT_MODE_STD_DEV = 'std_dev' | 191 BISECT_MODE_STD_DEV = 'std_dev' |
| 185 BISECT_MODE_RETURN_CODE = 'return_code' | 192 BISECT_MODE_RETURN_CODE = 'return_code' |
| 186 | 193 |
| 187 # The perf dashboard specifically looks for the string | 194 # The perf dashboard specifically looks for the string |
| 188 # "Estimated Confidence: 95%" to decide whether or not | 195 # "Estimated Confidence: 95%" to decide whether or not to cc the author(s). |
| 189 # to cc the author(s). If you change this, please update the perf | 196 # If you change this, please update the perf dashboard as well. |
| 190 # dashboard as well. | |
| 191 RESULTS_BANNER = """ | 197 RESULTS_BANNER = """ |
| 192 ===== BISECT JOB RESULTS ===== | 198 ===== BISECT JOB RESULTS ===== |
| 193 Status: %(status)s | 199 Status: %(status)s |
| 194 | 200 |
| 195 Test Command: %(command)s | 201 Test Command: %(command)s |
| 196 Test Metric: %(metrics)s | 202 Test Metric: %(metrics)s |
| 197 Relative Change: %(change)s | 203 Relative Change: %(change)s |
| 198 Estimated Confidence: %(confidence)d%%""" | 204 Estimated Confidence: %(confidence)d%%""" |
| 199 | 205 |
| 200 # The perf dashboard specifically looks for the string | 206 # The perf dashboard specifically looks for the string |
| (...skipping 22 matching lines...) Expand all Loading... |
| 223 --browser=android-chromium-testshell. | 229 --browser=android-chromium-testshell. |
| 224 c) Test command to use: %(command)s | 230 c) Test command to use: %(command)s |
| 225 3. Upload your patch. --bypass-hooks is necessary to upload the changes you \ | 231 3. Upload your patch. --bypass-hooks is necessary to upload the changes you \ |
| 226 committed locally to run-perf-test.cfg. | 232 committed locally to run-perf-test.cfg. |
| 227 Note: *DO NOT* commit run-perf-test.cfg changes to the project repository. | 233 Note: *DO NOT* commit run-perf-test.cfg changes to the project repository. |
| 228 $ git cl upload --bypass-hooks | 234 $ git cl upload --bypass-hooks |
| 229 4. Send your try job to the tryserver. \ | 235 4. Send your try job to the tryserver. \ |
| 230 [Please make sure to use appropriate bot to reproduce] | 236 [Please make sure to use appropriate bot to reproduce] |
| 231 $ git cl try -m tryserver.chromium.perf -b <bot> | 237 $ git cl try -m tryserver.chromium.perf -b <bot> |
| 232 | 238 |
| 233 For more details please visit \nhttps://sites.google.com/a/chromium.org/dev/\ | 239 For more details please visit |
| 234 developers/performance-try-bots""" | 240 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots""" |
| 235 | 241 |
| 236 RESULTS_THANKYOU = """ | 242 RESULTS_THANKYOU = """ |
| 237 ===== THANK YOU FOR CHOOSING BISECT AIRLINES ===== | 243 ===== THANK YOU FOR CHOOSING BISECT AIRLINES ===== |
| 238 Visit http://www.chromium.org/developers/core-principles for Chrome's policy | 244 Visit http://www.chromium.org/developers/core-principles for Chrome's policy |
| 239 on perf regressions. | 245 on perf regressions. |
| 240 Contact chrome-perf-dashboard-team with any questions or suggestions about | 246 Contact chrome-perf-dashboard-team with any questions or suggestions about |
| 241 bisecting. | 247 bisecting. |
| 242 . .------. | 248 . .------. |
| 243 . .---. \ \==) | 249 . .---. \ \==) |
| 244 . |PERF\ \ \\ | 250 . |PERF\ \ \\ |
| 245 . | ---------'-------'-----------. | 251 . | ---------'-------'-----------. |
| 246 . . 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 `-. | 252 . . 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 `-. |
| 247 . \______________.-------._______________) | 253 . \______________.-------._______________) |
| 248 . / / | 254 . / / |
| 249 . / / | 255 . / / |
| 250 . / /==) | 256 . / /==) |
| 251 . ._______.""" | 257 . ._______.""" |
| 252 | 258 |
| 253 | 259 |
| 254 def _AddAdditionalDepotInfo(depot_info): | 260 def _AddAdditionalDepotInfo(depot_info): |
| 255 """Adds additional depot info to the global depot variables.""" | 261 """Adds additional depot info to the global depot variables.""" |
| 256 global DEPOT_DEPS_NAME | 262 global DEPOT_DEPS_NAME |
| 257 global DEPOT_NAMES | 263 global DEPOT_NAMES |
| 258 DEPOT_DEPS_NAME = dict(DEPOT_DEPS_NAME.items() + | 264 DEPOT_DEPS_NAME = dict(DEPOT_DEPS_NAME.items() + depot_info.items()) |
| 259 depot_info.items()) | |
| 260 DEPOT_NAMES = DEPOT_DEPS_NAME.keys() | 265 DEPOT_NAMES = DEPOT_DEPS_NAME.keys() |
| 261 | 266 |
| 262 | 267 |
| 263 def ConfidenceScore(good_results_lists, bad_results_lists): | 268 def ConfidenceScore(good_results_lists, bad_results_lists): |
| 264 """Calculates a confidence score. | 269 """Calculates a confidence score. |
| 265 | 270 |
| 266 This score is a percentage which represents our degree of confidence in the | 271 This score is a percentage which represents our degree of confidence in the |
| 267 proposition that the good results and bad results are distinct groups, and | 272 proposition that the good results and bad results are distinct groups, and |
| 268 their differences aren't due to chance alone. | 273 their differences aren't due to chance alone. |
| 269 | 274 |
| (...skipping 12 matching lines...) Expand all Loading... |
| 282 sample1 = sum(good_results_lists, []) | 287 sample1 = sum(good_results_lists, []) |
| 283 sample2 = sum(bad_results_lists, []) | 288 sample2 = sum(bad_results_lists, []) |
| 284 | 289 |
| 285 # The p-value is approximately the probability of obtaining the given set | 290 # The p-value is approximately the probability of obtaining the given set |
| 286 # of good and bad values just by chance. | 291 # of good and bad values just by chance. |
| 287 _, _, p_value = ttest.WelchsTTest(sample1, sample2) | 292 _, _, p_value = ttest.WelchsTTest(sample1, sample2) |
| 288 return 100.0 * (1.0 - p_value) | 293 return 100.0 * (1.0 - p_value) |
| 289 | 294 |
| 290 | 295 |
| 291 def GetSHA1HexDigest(contents): | 296 def GetSHA1HexDigest(contents): |
| 292 """Returns secured hash containing hexadecimal for the given contents.""" | 297 """Returns SHA1 hex digest of the given string.""" |
| 293 return hashlib.sha1(contents).hexdigest() | 298 return hashlib.sha1(contents).hexdigest() |
| 294 | 299 |
| 295 | 300 |
| 296 def GetZipFileName(build_revision=None, target_arch='ia32', patch_sha=None): | 301 def GetZipFileName(build_revision=None, target_arch='ia32', patch_sha=None): |
| 297 """Gets the archive file name for the given revision.""" | 302 """Gets the archive file name for the given revision.""" |
| 298 def PlatformName(): | 303 def PlatformName(): |
| 299 """Return a string to be used in paths for the platform.""" | 304 """Return a string to be used in paths for the platform.""" |
| 300 if bisect_utils.IsWindowsHost(): | 305 if bisect_utils.IsWindowsHost(): |
| 301 # Build archive for x64 is still stored with 'win32'suffix | 306 # Build archive for x64 is still stored with 'win32'suffix |
| 302 # (chromium_utils.PlatformName()). | 307 # (chromium_utils.PlatformName()). |
| (...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 368 os.remove(target_file) | 373 os.remove(target_file) |
| 369 return None | 374 return None |
| 370 | 375 |
| 371 | 376 |
| 372 # This is copied from Chromium's project build/scripts/common/chromium_utils.py. | 377 # This is copied from Chromium's project build/scripts/common/chromium_utils.py. |
| 373 def MaybeMakeDirectory(*path): | 378 def MaybeMakeDirectory(*path): |
| 374 """Creates an entire path, if it doesn't already exist.""" | 379 """Creates an entire path, if it doesn't already exist.""" |
| 375 file_path = os.path.join(*path) | 380 file_path = os.path.join(*path) |
| 376 try: | 381 try: |
| 377 os.makedirs(file_path) | 382 os.makedirs(file_path) |
| 378 except OSError, e: | 383 except OSError as e: |
| 379 if e.errno != errno.EEXIST: | 384 if e.errno != errno.EEXIST: |
| 380 return False | 385 return False |
| 381 return True | 386 return True |
| 382 | 387 |
| 383 | 388 |
| 384 # This is copied from Chromium's project build/scripts/common/chromium_utils.py. | 389 # This is copied from Chromium's project build/scripts/common/chromium_utils.py. |
| 385 def ExtractZip(filename, output_dir, verbose=True): | 390 def ExtractZip(filename, output_dir, verbose=True): |
| 386 """ Extract the zip archive in the output directory.""" | 391 """ Extract the zip archive in the output directory.""" |
| 387 MaybeMakeDirectory(output_dir) | 392 MaybeMakeDirectory(output_dir) |
| 388 | 393 |
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 421 for name in zf.namelist(): | 426 for name in zf.namelist(): |
| 422 if verbose: | 427 if verbose: |
| 423 print 'Extracting %s' % name | 428 print 'Extracting %s' % name |
| 424 zf.extract(name, output_dir) | 429 zf.extract(name, output_dir) |
| 425 if bisect_utils.IsMacHost(): | 430 if bisect_utils.IsMacHost(): |
| 426 # Restore permission bits. | 431 # Restore permission bits. |
| 427 os.chmod(os.path.join(output_dir, name), | 432 os.chmod(os.path.join(output_dir, name), |
| 428 zf.getinfo(name).external_attr >> 16L) | 433 zf.getinfo(name).external_attr >> 16L) |
| 429 | 434 |
| 430 | 435 |
| 431 | |
| 432 | |
| 433 def SetBuildSystemDefault(build_system, use_goma, goma_dir): | 436 def SetBuildSystemDefault(build_system, use_goma, goma_dir): |
| 434 """Sets up any environment variables needed to build with the specified build | 437 """Sets up any environment variables needed to build with the specified build |
| 435 system. | 438 system. |
| 436 | 439 |
| 437 Args: | 440 Args: |
| 438 build_system: A string specifying build system. Currently only 'ninja' or | 441 build_system: A string specifying build system. Currently only 'ninja' or |
| 439 'make' are supported.""" | 442 'make' are supported. |
| 443 """ |
| 440 if build_system == 'ninja': | 444 if build_system == 'ninja': |
| 441 gyp_var = os.getenv('GYP_GENERATORS', default='') | 445 gyp_var = os.getenv('GYP_GENERATORS', default='') |
| 442 | 446 |
| 443 if not gyp_var or not 'ninja' in gyp_var: | 447 if not gyp_var or not 'ninja' in gyp_var: |
| 444 if gyp_var: | 448 if gyp_var: |
| 445 os.environ['GYP_GENERATORS'] = gyp_var + ',ninja' | 449 os.environ['GYP_GENERATORS'] = gyp_var + ',ninja' |
| 446 else: | 450 else: |
| 447 os.environ['GYP_GENERATORS'] = 'ninja' | 451 os.environ['GYP_GENERATORS'] = 'ninja' |
| 448 | 452 |
| 449 if bisect_utils.IsWindowsHost(): | 453 if bisect_utils.IsWindowsHost(): |
| 450 os.environ['GYP_DEFINES'] = 'component=shared_library '\ | 454 os.environ['GYP_DEFINES'] = ('component=shared_library ' |
| 451 'incremental_chrome_dll=1 disable_nacl=1 fastbuild=1 '\ | 455 'incremental_chrome_dll=1 ' |
| 452 'chromium_win_pch=0' | 456 'disable_nacl=1 fastbuild=1 ' |
| 457 'chromium_win_pch=0') |
| 453 | 458 |
| 454 elif build_system == 'make': | 459 elif build_system == 'make': |
| 455 os.environ['GYP_GENERATORS'] = 'make' | 460 os.environ['GYP_GENERATORS'] = 'make' |
| 456 else: | 461 else: |
| 457 raise RuntimeError('%s build not supported.' % build_system) | 462 raise RuntimeError('%s build not supported.' % build_system) |
| 458 | 463 |
| 459 if use_goma: | 464 if use_goma: |
| 460 os.environ['GYP_DEFINES'] = '%s %s' % (os.getenv('GYP_DEFINES', default=''), | 465 os.environ['GYP_DEFINES'] = '%s %s' % (os.getenv('GYP_DEFINES', default=''), |
| 461 'use_goma=1') | 466 'use_goma=1') |
| 462 if goma_dir: | 467 if goma_dir: |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 498 for t in targets: | 503 for t in targets: |
| 499 cmd.extend(['/Project', t]) | 504 cmd.extend(['/Project', t]) |
| 500 | 505 |
| 501 return_code = bisect_utils.RunProcess(cmd) | 506 return_code = bisect_utils.RunProcess(cmd) |
| 502 | 507 |
| 503 return not return_code | 508 return not return_code |
| 504 | 509 |
| 505 | 510 |
| 506 def WriteStringToFile(text, file_name): | 511 def WriteStringToFile(text, file_name): |
| 507 try: | 512 try: |
| 508 with open(file_name, "wb") as f: | 513 with open(file_name, 'wb') as f: |
| 509 f.write(text) | 514 f.write(text) |
| 510 except IOError as e: | 515 except IOError: |
| 511 raise RuntimeError('Error writing to file [%s]' % file_name ) | 516 raise RuntimeError('Error writing to file [%s]' % file_name ) |
| 512 | 517 |
| 513 | 518 |
| 514 def ReadStringFromFile(file_name): | 519 def ReadStringFromFile(file_name): |
| 515 try: | 520 try: |
| 516 with open(file_name) as f: | 521 with open(file_name) as f: |
| 517 return f.read() | 522 return f.read() |
| 518 except IOError as e: | 523 except IOError: |
| 519 raise RuntimeError('Error reading file [%s]' % file_name ) | 524 raise RuntimeError('Error reading file [%s]' % file_name ) |
| 520 | 525 |
| 521 | 526 |
| 522 def ChangeBackslashToSlashInPatch(diff_text): | 527 def ChangeBackslashToSlashInPatch(diff_text): |
| 523 """Formats file paths in the given text to unix-style paths.""" | 528 """Formats file paths in the given text to unix-style paths.""" |
| 524 if diff_text: | 529 if diff_text: |
| 525 diff_lines = diff_text.split('\n') | 530 diff_lines = diff_text.split('\n') |
| 526 for i in range(len(diff_lines)): | 531 for i in range(len(diff_lines)): |
| 527 if (diff_lines[i].startswith('--- ') or | 532 if (diff_lines[i].startswith('--- ') or |
| 528 diff_lines[i].startswith('+++ ')): | 533 diff_lines[i].startswith('+++ ')): |
| (...skipping 160 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 689 try: | 694 try: |
| 690 # Keys will most likely be set to 0640 after wiping the chroot. | 695 # Keys will most likely be set to 0640 after wiping the chroot. |
| 691 os.chmod(CROS_SCRIPT_KEY_PATH, 0600) | 696 os.chmod(CROS_SCRIPT_KEY_PATH, 0600) |
| 692 os.chmod(CROS_TEST_KEY_PATH, 0600) | 697 os.chmod(CROS_TEST_KEY_PATH, 0600) |
| 693 cmd = [CROS_SDK_PATH, '--', './bin/cros_image_to_target.py', | 698 cmd = [CROS_SDK_PATH, '--', './bin/cros_image_to_target.py', |
| 694 '--remote=%s' % opts.cros_remote_ip, | 699 '--remote=%s' % opts.cros_remote_ip, |
| 695 '--board=%s' % opts.cros_board, '--test', '--verbose'] | 700 '--board=%s' % opts.cros_board, '--test', '--verbose'] |
| 696 | 701 |
| 697 return_code = bisect_utils.RunProcess(cmd) | 702 return_code = bisect_utils.RunProcess(cmd) |
| 698 return not return_code | 703 return not return_code |
| 699 except OSError, e: | 704 except OSError: |
| 700 return False | 705 return False |
| 701 | 706 |
| 702 def BuildPackages(self, opts, depot): | 707 def BuildPackages(self, opts, depot): |
| 703 """Builds packages for cros. | 708 """Builds packages for cros. |
| 704 | 709 |
| 705 Args: | 710 Args: |
| 706 opts: Program options containing cros_board. | 711 opts: Program options containing cros_board. |
| 707 depot: The depot being bisected. | 712 depot: The depot being bisected. |
| 708 | 713 |
| 709 Returns: | 714 Returns: |
| (...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 763 | 768 |
| 764 Returns: | 769 Returns: |
| 765 True if build was successful. | 770 True if build was successful. |
| 766 """ | 771 """ |
| 767 if self.BuildPackages(opts, depot): | 772 if self.BuildPackages(opts, depot): |
| 768 if self.BuildImage(opts, depot): | 773 if self.BuildImage(opts, depot): |
| 769 return self.ImageToTarget(opts) | 774 return self.ImageToTarget(opts) |
| 770 return False | 775 return False |
| 771 | 776 |
| 772 | 777 |
| 773 | 778 def _ParseRevisionsFromDEPSFileManually(deps_file_contents): |
| 779 """Parses the vars section of the DEPS file with regex. |
| 780 |
| 781 Args: |
| 782 deps_file_contents: The DEPS file contents as a string. |
| 783 |
| 784 Returns: |
| 785 A dict in the format {depot:revision} if successful, otherwise None. |
| 786 """ |
| 787 # We'll parse the "vars" section of the DEPS file. |
| 788 rxp = re.compile('vars = {(?P<vars_body>[^}]+)', re.MULTILINE) |
| 789 re_results = rxp.search(deps_file_contents) |
| 790 |
| 791 if not re_results: |
| 792 return None |
| 793 |
| 794 # We should be left with a series of entries in the vars component of |
| 795 # the DEPS file with the following format: |
| 796 # 'depot_name': 'revision', |
| 797 vars_body = re_results.group('vars_body') |
| 798 rxp = re.compile("'(?P<depot_body>[\w_-]+)':[\s]+'(?P<rev_body>[\w@]+)'", |
| 799 re.MULTILINE) |
| 800 re_results = rxp.findall(vars_body) |
| 801 |
| 802 return dict(re_results) |
| 803 |
| 804 |
| 805 def _WaitUntilBuildIsReady( |
| 806 fetch_build, bot_name, builder_host, builder_port, build_request_id, |
| 807 max_timeout): |
| 808 """Waits until build is produced by bisect builder on tryserver. |
| 809 |
| 810 Args: |
| 811 fetch_build: Function to check and download build from cloud storage. |
| 812 bot_name: Builder bot name on tryserver. |
| 813 builder_host Tryserver hostname. |
| 814 builder_port: Tryserver port. |
| 815 build_request_id: A unique ID of the build request posted to tryserver. |
| 816 max_timeout: Maximum time to wait for the build. |
| 817 |
| 818 Returns: |
| 819 Downloaded archive file path if exists, otherwise None. |
| 820 """ |
| 821 # Build number on the tryserver. |
| 822 build_num = None |
| 823 # Interval to check build on cloud storage. |
| 824 poll_interval = 60 |
| 825 # Interval to check build status on tryserver. |
| 826 status_check_interval = 600 |
| 827 last_status_check = time.time() |
| 828 start_time = time.time() |
| 829 while True: |
| 830 # Checks for build on gs://chrome-perf and download if exists. |
| 831 res = fetch_build() |
| 832 if res: |
| 833 return (res, 'Build successfully found') |
| 834 elapsed_status_check = time.time() - last_status_check |
| 835 # To avoid overloading tryserver with status check requests, we check |
| 836 # build status for every 10 mins. |
| 837 if elapsed_status_check > status_check_interval: |
| 838 last_status_check = time.time() |
| 839 if not build_num: |
| 840 # Get the build number on tryserver for the current build. |
| 841 build_num = bisect_builder.GetBuildNumFromBuilder( |
| 842 build_request_id, bot_name, builder_host, builder_port) |
| 843 # Check the status of build using the build number. |
| 844 # Note: Build is treated as PENDING if build number is not found |
| 845 # on the the tryserver. |
| 846 build_status, status_link = bisect_builder.GetBuildStatus( |
| 847 build_num, bot_name, builder_host, builder_port) |
| 848 if build_status == bisect_builder.FAILED: |
| 849 return (None, 'Failed to produce build, log: %s' % status_link) |
| 850 elapsed_time = time.time() - start_time |
| 851 if elapsed_time > max_timeout: |
| 852 return (None, 'Timed out: %ss without build' % max_timeout) |
| 853 |
| 854 print 'Time elapsed: %ss without build.' % elapsed_time |
| 855 time.sleep(poll_interval) |
| 856 # For some reason, mac bisect bots were not flushing stdout periodically. |
| 857 # As a result buildbot command is timed-out. Flush stdout on all platforms |
| 858 # while waiting for build. |
| 859 sys.stdout.flush() |
| 860 |
| 861 |
| 862 def _UpdateV8Branch(deps_content): |
| 863 """Updates V8 branch in DEPS file to process v8_bleeding_edge. |
| 864 |
| 865 Check for "v8_branch" in DEPS file if exists update its value |
| 866 with v8_bleeding_edge branch. Note: "v8_branch" is added to DEPS |
| 867 variable from DEPS revision 254916, therefore check for "src/v8": |
| 868 <v8 source path> in DEPS in order to support prior DEPS revisions |
| 869 and update it. |
| 870 |
| 871 Args: |
| 872 deps_content: DEPS file contents to be modified. |
| 873 |
| 874 Returns: |
| 875 Modified DEPS file contents as a string. |
| 876 """ |
| 877 new_branch = r'branches/bleeding_edge' |
| 878 v8_branch_pattern = re.compile(r'(?<="v8_branch": ")(.*)(?=")') |
| 879 if re.search(v8_branch_pattern, deps_content): |
| 880 deps_content = re.sub(v8_branch_pattern, new_branch, deps_content) |
| 881 else: |
| 882 # Replaces the branch assigned to "src/v8" key in DEPS file. |
| 883 # Format of "src/v8" in DEPS: |
| 884 # "src/v8": |
| 885 # (Var("googlecode_url") % "v8") + "/trunk@" + Var("v8_revision"), |
| 886 # So, "/trunk@" is replace with "/branches/bleeding_edge@" |
| 887 v8_src_pattern = re.compile( |
| 888 r'(?<="v8"\) \+ "/)(.*)(?=@" \+ Var\("v8_revision"\))', re.MULTILINE) |
| 889 if re.search(v8_src_pattern, deps_content): |
| 890 deps_content = re.sub(v8_src_pattern, new_branch, deps_content) |
| 891 return deps_content |
| 892 |
| 893 |
| 894 def _UpdateDEPSForAngle(revision, depot, deps_file): |
| 895 """Updates DEPS file with new revision for Angle repository. |
| 896 |
| 897 This is a hack for Angle depot case because, in DEPS file "vars" dictionary |
| 898 variable contains "angle_revision" key that holds git hash instead of |
| 899 SVN revision. |
| 900 |
| 901 And sometimes "angle_revision" key is not specified in "vars" variable, |
| 902 in such cases check "deps" dictionary variable that matches |
| 903 angle.git@[a-fA-F0-9]{40}$ and replace git hash. |
| 904 """ |
| 905 deps_var = DEPOT_DEPS_NAME[depot]['deps_var'] |
| 906 try: |
| 907 deps_contents = ReadStringFromFile(deps_file) |
| 908 # Check whether the depot and revision pattern in DEPS file vars variable |
| 909 # e.g. "angle_revision": "fa63e947cb3eccf463648d21a05d5002c9b8adfa". |
| 910 angle_rev_pattern = re.compile(r'(?<="%s": ")([a-fA-F0-9]{40})(?=")' % |
| 911 deps_var, re.MULTILINE) |
| 912 match = re.search(angle_rev_pattern % deps_var, deps_contents) |
| 913 if match: |
| 914 # Update the revision information for the given depot |
| 915 new_data = re.sub(angle_rev_pattern, revision, deps_contents) |
| 916 else: |
| 917 # Check whether the depot and revision pattern in DEPS file deps |
| 918 # variable. e.g., |
| 919 # "src/third_party/angle": Var("chromium_git") + |
| 920 # "/angle/angle.git@fa63e947cb3eccf463648d21a05d5002c9b8adfa",. |
| 921 angle_rev_pattern = re.compile( |
| 922 r'(?<=angle\.git@)([a-fA-F0-9]{40})(?=")', re.MULTILINE) |
| 923 match = re.search(angle_rev_pattern, deps_contents) |
| 924 if not match: |
| 925 print 'Could not find angle revision information in DEPS file.' |
| 926 return False |
| 927 new_data = re.sub(angle_rev_pattern, revision, deps_contents) |
| 928 # Write changes to DEPS file |
| 929 WriteStringToFile(new_data, deps_file) |
| 930 return True |
| 931 except IOError, e: |
| 932 print 'Something went wrong while updating DEPS file, %s' % e |
| 933 return False |
| 934 |
| 935 |
| 936 def _TryParseHistogramValuesFromOutput(metric, text): |
| 937 """Attempts to parse a metric in the format HISTOGRAM <graph: <trace>. |
| 938 |
| 939 Args: |
| 940 metric: The metric as a list of [<trace>, <value>] strings. |
| 941 text: The text to parse the metric values from. |
| 942 |
| 943 Returns: |
| 944 A list of floating point numbers found, [] if none were found. |
| 945 """ |
| 946 metric_formatted = 'HISTOGRAM %s: %s= ' % (metric[0], metric[1]) |
| 947 |
| 948 text_lines = text.split('\n') |
| 949 values_list = [] |
| 950 |
| 951 for current_line in text_lines: |
| 952 if metric_formatted in current_line: |
| 953 current_line = current_line[len(metric_formatted):] |
| 954 |
| 955 try: |
| 956 histogram_values = eval(current_line) |
| 957 |
| 958 for b in histogram_values['buckets']: |
| 959 average_for_bucket = float(b['high'] + b['low']) * 0.5 |
| 960 # Extends the list with N-elements with the average for that bucket. |
| 961 values_list.extend([average_for_bucket] * b['count']) |
| 962 except Exception: |
| 963 pass |
| 964 |
| 965 return values_list |
| 966 |
| 967 |
| 968 def _TryParseResultValuesFromOutput(metric, text): |
| 969 """Attempts to parse a metric in the format RESULT <graph>: <trace>= ... |
| 970 |
| 971 Args: |
| 972 metric: The metric as a list of [<trace>, <value>] string pairs. |
| 973 text: The text to parse the metric values from. |
| 974 |
| 975 Returns: |
| 976 A list of floating point numbers found. |
| 977 """ |
| 978 # Format is: RESULT <graph>: <trace>= <value> <units> |
| 979 metric_re = re.escape('RESULT %s: %s=' % (metric[0], metric[1])) |
| 980 |
| 981 # The log will be parsed looking for format: |
| 982 # <*>RESULT <graph_name>: <trace_name>= <value> |
| 983 single_result_re = re.compile( |
| 984 metric_re + '\s*(?P<VALUE>[-]?\d*(\.\d*)?)') |
| 985 |
| 986 # The log will be parsed looking for format: |
| 987 # <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...] |
| 988 multi_results_re = re.compile( |
| 989 metric_re + '\s*\[\s*(?P<VALUES>[-]?[\d\., ]+)\s*\]') |
| 990 |
| 991 # The log will be parsed looking for format: |
| 992 # <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} |
| 993 mean_stddev_re = re.compile( |
| 994 metric_re + |
| 995 '\s*\{\s*(?P<MEAN>[-]?\d*(\.\d*)?),\s*(?P<STDDEV>\d+(\.\d*)?)\s*\}') |
| 996 |
| 997 text_lines = text.split('\n') |
| 998 values_list = [] |
| 999 for current_line in text_lines: |
| 1000 # Parse the output from the performance test for the metric we're |
| 1001 # interested in. |
| 1002 single_result_match = single_result_re.search(current_line) |
| 1003 multi_results_match = multi_results_re.search(current_line) |
| 1004 mean_stddev_match = mean_stddev_re.search(current_line) |
| 1005 if (not single_result_match is None and |
| 1006 single_result_match.group('VALUE')): |
| 1007 values_list += [single_result_match.group('VALUE')] |
| 1008 elif (not multi_results_match is None and |
| 1009 multi_results_match.group('VALUES')): |
| 1010 metric_values = multi_results_match.group('VALUES') |
| 1011 values_list += metric_values.split(',') |
| 1012 elif (not mean_stddev_match is None and |
| 1013 mean_stddev_match.group('MEAN')): |
| 1014 values_list += [mean_stddev_match.group('MEAN')] |
| 1015 |
| 1016 values_list = [float(v) for v in values_list |
| 1017 if bisect_utils.IsStringFloat(v)] |
| 1018 |
| 1019 # If the metric is times/t, we need to sum the timings in order to get |
| 1020 # similar regression results as the try-bots. |
| 1021 metrics_to_sum = [ |
| 1022 ['times', 't'], |
| 1023 ['times', 'page_load_time'], |
| 1024 ['cold_times', 'page_load_time'], |
| 1025 ['warm_times', 'page_load_time'], |
| 1026 ] |
| 1027 |
| 1028 if metric in metrics_to_sum: |
| 1029 if values_list: |
| 1030 values_list = [reduce(lambda x, y: float(x) + float(y), values_list)] |
| 1031 |
| 1032 return values_list |
| 1033 |
| 1034 |
| 1035 def _ParseMetricValuesFromOutput(metric, text): |
| 1036 """Parses output from performance_ui_tests and retrieves the results for |
| 1037 a given metric. |
| 1038 |
| 1039 Args: |
| 1040 metric: The metric as a list of [<trace>, <value>] strings. |
| 1041 text: The text to parse the metric values from. |
| 1042 |
| 1043 Returns: |
| 1044 A list of floating point numbers found. |
| 1045 """ |
| 1046 metric_values = _TryParseResultValuesFromOutput(metric, text) |
| 1047 |
| 1048 if not metric_values: |
| 1049 metric_values = _TryParseHistogramValuesFromOutput(metric, text) |
| 1050 |
| 1051 return metric_values |
| 1052 |
| 1053 |
| 1054 def _GenerateProfileIfNecessary(command_args): |
| 1055 """Checks the command line of the performance test for dependencies on |
| 1056 profile generation, and runs tools/perf/generate_profile as necessary. |
| 1057 |
| 1058 Args: |
| 1059 command_args: Command line being passed to performance test, as a list. |
| 1060 |
| 1061 Returns: |
| 1062 False if profile generation was necessary and failed, otherwise True. |
| 1063 """ |
| 1064 if '--profile-dir' in ' '.join(command_args): |
| 1065 # If we were using python 2.7+, we could just use the argparse |
| 1066 # module's parse_known_args to grab --profile-dir. Since some of the |
| 1067 # bots still run 2.6, have to grab the arguments manually. |
| 1068 arg_dict = {} |
| 1069 args_to_parse = ['--profile-dir', '--browser'] |
| 1070 |
| 1071 for arg_to_parse in args_to_parse: |
| 1072 for i, current_arg in enumerate(command_args): |
| 1073 if arg_to_parse in current_arg: |
| 1074 current_arg_split = current_arg.split('=') |
| 1075 |
| 1076 # Check 2 cases, --arg=<val> and --arg <val> |
| 1077 if len(current_arg_split) == 2: |
| 1078 arg_dict[arg_to_parse] = current_arg_split[1] |
| 1079 elif i + 1 < len(command_args): |
| 1080 arg_dict[arg_to_parse] = command_args[i+1] |
| 1081 |
| 1082 path_to_generate = os.path.join('tools', 'perf', 'generate_profile') |
| 1083 |
| 1084 if arg_dict.has_key('--profile-dir') and arg_dict.has_key('--browser'): |
| 1085 profile_path, profile_type = os.path.split(arg_dict['--profile-dir']) |
| 1086 return not bisect_utils.RunProcess(['python', path_to_generate, |
| 1087 '--profile-type-to-generate', profile_type, |
| 1088 '--browser', arg_dict['--browser'], '--output-dir', profile_path]) |
| 1089 return False |
| 1090 return True |
| 1091 |
| 1092 |
| 1093 def _AddRevisionsIntoRevisionData(revisions, depot, sort, revision_data): |
| 1094 """Adds new revisions to the revision_data dict and initializes them. |
| 1095 |
| 1096 Args: |
| 1097 revisions: List of revisions to add. |
| 1098 depot: Depot that's currently in use (src, webkit, etc...) |
| 1099 sort: Sorting key for displaying revisions. |
| 1100 revision_data: A dict to add the new revisions into. Existing revisions |
| 1101 will have their sort keys offset. |
| 1102 """ |
| 1103 num_depot_revisions = len(revisions) |
| 1104 |
| 1105 for _, v in revision_data.iteritems(): |
| 1106 if v['sort'] > sort: |
| 1107 v['sort'] += num_depot_revisions |
| 1108 |
| 1109 for i in xrange(num_depot_revisions): |
| 1110 r = revisions[i] |
| 1111 revision_data[r] = { |
| 1112 'revision' : r, |
| 1113 'depot' : depot, |
| 1114 'value' : None, |
| 1115 'perf_time' : 0, |
| 1116 'build_time' : 0, |
| 1117 'passed' : '?', |
| 1118 'sort' : i + sort + 1, |
| 1119 } |
| 1120 |
| 1121 |
| 1122 def _PrintThankYou(): |
| 1123 print RESULTS_THANKYOU |
| 1124 |
| 1125 |
| 1126 def _PrintTableRow(column_widths, row_data): |
| 1127 """Prints out a row in a formatted table that has columns aligned. |
| 1128 |
| 1129 Args: |
| 1130 column_widths: A list of column width numbers. |
| 1131 row_data: A list of items for each column in this row. |
| 1132 """ |
| 1133 assert len(column_widths) == len(row_data) |
| 1134 text = '' |
| 1135 for i in xrange(len(column_widths)): |
| 1136 current_row_data = row_data[i].center(column_widths[i], ' ') |
| 1137 text += ('%%%ds' % column_widths[i]) % current_row_data |
| 1138 print text |
| 1139 |
| 1140 |
| 1141 def _PrintStepTime(revision_data_sorted): |
| 1142 """Prints information about how long various steps took. |
| 1143 |
| 1144 Args: |
| 1145 revision_data_sorted: The sorted list of revision data dictionaries.""" |
| 1146 step_perf_time_avg = 0.0 |
| 1147 step_build_time_avg = 0.0 |
| 1148 step_count = 0.0 |
| 1149 for _, current_data in revision_data_sorted: |
| 1150 if current_data['value']: |
| 1151 step_perf_time_avg += current_data['perf_time'] |
| 1152 step_build_time_avg += current_data['build_time'] |
| 1153 step_count += 1 |
| 1154 if step_count: |
| 1155 step_perf_time_avg = step_perf_time_avg / step_count |
| 1156 step_build_time_avg = step_build_time_avg / step_count |
| 1157 print |
| 1158 print 'Average build time : %s' % datetime.timedelta( |
| 1159 seconds=int(step_build_time_avg)) |
| 1160 print 'Average test time : %s' % datetime.timedelta( |
| 1161 seconds=int(step_perf_time_avg)) |
| 1162 |
| 1163 def _FindOtherRegressions(revision_data_sorted, bad_greater_than_good): |
| 1164 """Compiles a list of other possible regressions from the revision data. |
| 1165 |
| 1166 Args: |
| 1167 revision_data_sorted: Sorted list of (revision, revision data dict) pairs. |
| 1168 bad_greater_than_good: Whether the result value at the "bad" revision is |
| 1169 numerically greater than the result value at the "good" revision. |
| 1170 |
| 1171 Returns: |
| 1172 A list of [current_rev, previous_rev, confidence] for other places where |
| 1173 there may have been a regression. |
| 1174 """ |
| 1175 other_regressions = [] |
| 1176 previous_values = [] |
| 1177 previous_id = None |
| 1178 for current_id, current_data in revision_data_sorted: |
| 1179 current_values = current_data['value'] |
| 1180 if current_values: |
| 1181 current_values = current_values['values'] |
| 1182 if previous_values: |
| 1183 confidence = ConfidenceScore(previous_values, [current_values]) |
| 1184 mean_of_prev_runs = math_utils.Mean(sum(previous_values, [])) |
| 1185 mean_of_current_runs = math_utils.Mean(current_values) |
| 1186 |
| 1187 # Check that the potential regression is in the same direction as |
| 1188 # the overall regression. If the mean of the previous runs < the |
| 1189 # mean of the current runs, this local regression is in same |
| 1190 # direction. |
| 1191 prev_less_than_current = mean_of_prev_runs < mean_of_current_runs |
| 1192 is_same_direction = (prev_less_than_current if |
| 1193 bad_greater_than_good else not prev_less_than_current) |
| 1194 |
| 1195 # Only report potential regressions with high confidence. |
| 1196 if is_same_direction and confidence > 50: |
| 1197 other_regressions.append([current_id, previous_id, confidence]) |
| 1198 previous_values.append(current_values) |
| 1199 previous_id = current_id |
| 1200 return other_regressions |
| 774 | 1201 |
| 775 class BisectPerformanceMetrics(object): | 1202 class BisectPerformanceMetrics(object): |
| 776 """This class contains functionality to perform a bisection of a range of | 1203 """This class contains functionality to perform a bisection of a range of |
| 777 revisions to narrow down where performance regressions may have occurred. | 1204 revisions to narrow down where performance regressions may have occurred. |
| 778 | 1205 |
| 779 The main entry-point is the Run method. | 1206 The main entry-point is the Run method. |
| 780 """ | 1207 """ |
| 781 | 1208 |
| 782 def __init__(self, source_control, opts): | 1209 def __init__(self, source_control, opts): |
| 783 super(BisectPerformanceMetrics, self).__init__() | 1210 super(BisectPerformanceMetrics, self).__init__() |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 823 cwd = os.getcwd() | 1250 cwd = os.getcwd() |
| 824 self.ChangeToDepotWorkingDirectory('cros') | 1251 self.ChangeToDepotWorkingDirectory('cros') |
| 825 | 1252 |
| 826 # Print the commit timestamps for every commit in the revision time | 1253 # Print the commit timestamps for every commit in the revision time |
| 827 # range. We'll sort them and bisect by that. There is a remote chance that | 1254 # range. We'll sort them and bisect by that. There is a remote chance that |
| 828 # 2 (or more) commits will share the exact same timestamp, but it's | 1255 # 2 (or more) commits will share the exact same timestamp, but it's |
| 829 # probably safe to ignore that case. | 1256 # probably safe to ignore that case. |
| 830 cmd = ['repo', 'forall', '-c', | 1257 cmd = ['repo', 'forall', '-c', |
| 831 'git log --format=%%ct --before=%d --after=%d' % ( | 1258 'git log --format=%%ct --before=%d --after=%d' % ( |
| 832 revision_range_end, revision_range_start)] | 1259 revision_range_end, revision_range_start)] |
| 833 (output, return_code) = bisect_utils.RunProcessAndRetrieveOutput(cmd) | 1260 output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd) |
| 834 | 1261 |
| 835 assert not return_code, 'An error occurred while running'\ | 1262 assert not return_code, ('An error occurred while running ' |
| 836 ' "%s"' % ' '.join(cmd) | 1263 '"%s"' % ' '.join(cmd)) |
| 837 | 1264 |
| 838 os.chdir(cwd) | 1265 os.chdir(cwd) |
| 839 | 1266 |
| 840 revision_work_list = list(set( | 1267 revision_work_list = list(set( |
| 841 [int(o) for o in output.split('\n') if bisect_utils.IsStringInt(o)])) | 1268 [int(o) for o in output.split('\n') if bisect_utils.IsStringInt(o)])) |
| 842 revision_work_list = sorted(revision_work_list, reverse=True) | 1269 revision_work_list = sorted(revision_work_list, reverse=True) |
| 843 else: | 1270 else: |
| 844 cwd = self._GetDepotDirectory(depot) | 1271 cwd = self._GetDepotDirectory(depot) |
| 845 revision_work_list = self.source_control.GetRevisionList(bad_revision, | 1272 revision_work_list = self.source_control.GetRevisionList(bad_revision, |
| 846 good_revision, cwd=cwd) | 1273 good_revision, cwd=cwd) |
| (...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 924 | 1351 |
| 925 bleeding_edge_revision = None | 1352 bleeding_edge_revision = None |
| 926 | 1353 |
| 927 for c in commits: | 1354 for c in commits: |
| 928 bleeding_edge_revision = self._GetV8BleedingEdgeFromV8TrunkIfMappable(c) | 1355 bleeding_edge_revision = self._GetV8BleedingEdgeFromV8TrunkIfMappable(c) |
| 929 if bleeding_edge_revision: | 1356 if bleeding_edge_revision: |
| 930 break | 1357 break |
| 931 | 1358 |
| 932 return bleeding_edge_revision | 1359 return bleeding_edge_revision |
| 933 | 1360 |
| 934 def _ParseRevisionsFromDEPSFileManually(self, deps_file_contents): | |
| 935 """Manually parses the vars section of the DEPS file to determine | |
| 936 chromium/blink/etc... revisions. | |
| 937 | |
| 938 Returns: | |
| 939 A dict in the format {depot:revision} if successful, otherwise None. | |
| 940 """ | |
| 941 # We'll parse the "vars" section of the DEPS file. | |
| 942 rxp = re.compile('vars = {(?P<vars_body>[^}]+)', re.MULTILINE) | |
| 943 re_results = rxp.search(deps_file_contents) | |
| 944 locals = {} | |
| 945 | |
| 946 if not re_results: | |
| 947 return None | |
| 948 | |
| 949 # We should be left with a series of entries in the vars component of | |
| 950 # the DEPS file with the following format: | |
| 951 # 'depot_name': 'revision', | |
| 952 vars_body = re_results.group('vars_body') | |
| 953 rxp = re.compile("'(?P<depot_body>[\w_-]+)':[\s]+'(?P<rev_body>[\w@]+)'", | |
| 954 re.MULTILINE) | |
| 955 re_results = rxp.findall(vars_body) | |
| 956 | |
| 957 return dict(re_results) | |
| 958 | |
| 959 def _ParseRevisionsFromDEPSFile(self, depot): | 1361 def _ParseRevisionsFromDEPSFile(self, depot): |
| 960 """Parses the local DEPS file to determine blink/skia/v8 revisions which may | 1362 """Parses the local DEPS file to determine blink/skia/v8 revisions which may |
| 961 be needed if the bisect recurses into those depots later. | 1363 be needed if the bisect recurses into those depots later. |
| 962 | 1364 |
| 963 Args: | 1365 Args: |
| 964 depot: Depot being bisected. | 1366 depot: Name of depot being bisected. |
| 965 | 1367 |
| 966 Returns: | 1368 Returns: |
| 967 A dict in the format {depot:revision} if successful, otherwise None. | 1369 A dict in the format {depot:revision} if successful, otherwise None. |
| 968 """ | 1370 """ |
| 969 try: | 1371 try: |
| 970 deps_data = {'Var': lambda _: deps_data["vars"][_], | 1372 deps_data = { |
| 971 'From': lambda *args: None | 1373 'Var': lambda _: deps_data["vars"][_], |
| 972 } | 1374 'From': lambda *args: None, |
| 1375 } |
| 973 execfile(bisect_utils.FILE_DEPS_GIT, {}, deps_data) | 1376 execfile(bisect_utils.FILE_DEPS_GIT, {}, deps_data) |
| 974 deps_data = deps_data['deps'] | 1377 deps_data = deps_data['deps'] |
| 975 | 1378 |
| 976 rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)") | 1379 rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)") |
| 977 results = {} | 1380 results = {} |
| 978 for depot_name, depot_data in DEPOT_DEPS_NAME.iteritems(): | 1381 for depot_name, depot_data in DEPOT_DEPS_NAME.iteritems(): |
| 979 if (depot_data.get('platform') and | 1382 if (depot_data.get('platform') and |
| 980 depot_data.get('platform') != os.name): | 1383 depot_data.get('platform') != os.name): |
| 981 continue | 1384 continue |
| 982 | 1385 |
| 983 if (depot_data.get('recurse') and depot in depot_data.get('from')): | 1386 if (depot_data.get('recurse') and depot in depot_data.get('from')): |
| 984 depot_data_src = depot_data.get('src') or depot_data.get('src_old') | 1387 depot_data_src = depot_data.get('src') or depot_data.get('src_old') |
| 985 src_dir = deps_data.get(depot_data_src) | 1388 src_dir = deps_data.get(depot_data_src) |
| 986 if src_dir: | 1389 if src_dir: |
| 987 self.depot_cwd[depot_name] = os.path.join(self.src_cwd, | 1390 self.depot_cwd[depot_name] = os.path.join(self.src_cwd, |
| 988 depot_data_src[4:]) | 1391 depot_data_src[4:]) |
| 989 re_results = rxp.search(src_dir) | 1392 re_results = rxp.search(src_dir) |
| 990 if re_results: | 1393 if re_results: |
| 991 results[depot_name] = re_results.group('revision') | 1394 results[depot_name] = re_results.group('revision') |
| 992 else: | 1395 else: |
| 993 warning_text = ('Couldn\'t parse revision for %s while bisecting ' | 1396 warning_text = ('Couldn\'t parse revision for %s while bisecting ' |
| 994 '%s' % (depot_name, depot)) | 1397 '%s' % (depot_name, depot)) |
| 995 if not warning_text in self.warnings: | 1398 if not warning_text in self.warnings: |
| 996 self.warnings.append(warning_text) | 1399 self.warnings.append(warning_text) |
| 997 else: | 1400 else: |
| 998 results[depot_name] = None | 1401 results[depot_name] = None |
| 999 return results | 1402 return results |
| 1000 except ImportError: | 1403 except ImportError: |
| 1001 deps_file_contents = ReadStringFromFile(bisect_utils.FILE_DEPS_GIT) | 1404 deps_file_contents = ReadStringFromFile(bisect_utils.FILE_DEPS_GIT) |
| 1002 parse_results = self._ParseRevisionsFromDEPSFileManually( | 1405 parse_results = _ParseRevisionsFromDEPSFileManually(deps_file_contents) |
| 1003 deps_file_contents) | |
| 1004 results = {} | 1406 results = {} |
| 1005 for depot_name, depot_revision in parse_results.iteritems(): | 1407 for depot_name, depot_revision in parse_results.iteritems(): |
| 1006 depot_revision = depot_revision.strip('@') | 1408 depot_revision = depot_revision.strip('@') |
| 1007 print depot_name, depot_revision | 1409 print depot_name, depot_revision |
| 1008 for current_name, current_data in DEPOT_DEPS_NAME.iteritems(): | 1410 for current_name, current_data in DEPOT_DEPS_NAME.iteritems(): |
| 1009 if (current_data.has_key('deps_var') and | 1411 if (current_data.has_key('deps_var') and |
| 1010 current_data['deps_var'] == depot_name): | 1412 current_data['deps_var'] == depot_name): |
| 1011 src_name = current_name | 1413 src_name = current_name |
| 1012 results[src_name] = depot_revision | 1414 results[src_name] = depot_revision |
| 1013 break | 1415 break |
| 1014 return results | 1416 return results |
| 1015 | 1417 |
| 1016 def Get3rdPartyRevisionsFromCurrentRevision(self, depot, revision): | 1418 def _Get3rdPartyRevisions(self, depot): |
| 1017 """Parses the DEPS file to determine WebKit/v8/etc... versions. | 1419 """Parses the DEPS file to determine WebKit/v8/etc... versions. |
| 1018 | 1420 |
| 1019 Returns: | 1421 Returns: |
| 1020 A dict in the format {depot:revision} if successful, otherwise None. | 1422 A dict in the format {depot:revision} if successful, otherwise None. |
| 1021 """ | 1423 """ |
| 1022 cwd = os.getcwd() | 1424 cwd = os.getcwd() |
| 1023 self.ChangeToDepotWorkingDirectory(depot) | 1425 self.ChangeToDepotWorkingDirectory(depot) |
| 1024 | 1426 |
| 1025 results = {} | 1427 results = {} |
| 1026 | 1428 |
| 1027 if depot == 'chromium' or depot == 'android-chrome': | 1429 if depot == 'chromium' or depot == 'android-chrome': |
| 1028 results = self._ParseRevisionsFromDEPSFile(depot) | 1430 results = self._ParseRevisionsFromDEPSFile(depot) |
| 1029 os.chdir(cwd) | 1431 os.chdir(cwd) |
| 1030 elif depot == 'cros': | 1432 elif depot == 'cros': |
| 1031 cmd = [CROS_SDK_PATH, '--', 'portageq-%s' % self.opts.cros_board, | 1433 cmd = [CROS_SDK_PATH, '--', 'portageq-%s' % self.opts.cros_board, |
| 1032 'best_visible', '/build/%s' % self.opts.cros_board, 'ebuild', | 1434 'best_visible', '/build/%s' % self.opts.cros_board, 'ebuild', |
| 1033 CROS_CHROMEOS_PATTERN] | 1435 CROS_CHROMEOS_PATTERN] |
| 1034 (output, return_code) = bisect_utils.RunProcessAndRetrieveOutput(cmd) | 1436 output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd) |
| 1035 | 1437 |
| 1036 assert not return_code, 'An error occurred while running' \ | 1438 assert not return_code, ('An error occurred while running ' |
| 1037 ' "%s"' % ' '.join(cmd) | 1439 '"%s"' % ' '.join(cmd)) |
| 1038 | 1440 |
| 1039 if len(output) > CROS_CHROMEOS_PATTERN: | 1441 if len(output) > CROS_CHROMEOS_PATTERN: |
| 1040 output = output[len(CROS_CHROMEOS_PATTERN):] | 1442 output = output[len(CROS_CHROMEOS_PATTERN):] |
| 1041 | 1443 |
| 1042 if len(output) > 1: | 1444 if len(output) > 1: |
| 1043 output = output.split('_')[0] | 1445 output = output.split('_')[0] |
| 1044 | 1446 |
| 1045 if len(output) > 3: | 1447 if len(output) > 3: |
| 1046 contents = output.split('.') | 1448 contents = output.split('.') |
| 1047 | 1449 |
| 1048 version = contents[2] | 1450 version = contents[2] |
| 1049 | 1451 |
| 1050 if contents[3] != '0': | 1452 if contents[3] != '0': |
| 1051 warningText = 'Chrome version: %s.%s but using %s.0 to bisect.' % \ | 1453 warningText = ('Chrome version: %s.%s but using %s.0 to bisect.' % |
| 1052 (version, contents[3], version) | 1454 (version, contents[3], version)) |
| 1053 if not warningText in self.warnings: | 1455 if not warningText in self.warnings: |
| 1054 self.warnings.append(warningText) | 1456 self.warnings.append(warningText) |
| 1055 | 1457 |
| 1056 cwd = os.getcwd() | 1458 cwd = os.getcwd() |
| 1057 self.ChangeToDepotWorkingDirectory('chromium') | 1459 self.ChangeToDepotWorkingDirectory('chromium') |
| 1058 cmd = ['log', '-1', '--format=%H', | 1460 cmd = ['log', '-1', '--format=%H', |
| 1059 '--author=chrome-release@google.com', | 1461 '--author=chrome-release@google.com', |
| 1060 '--grep=to %s' % version, 'origin/master'] | 1462 '--grep=to %s' % version, 'origin/master'] |
| 1061 return_code = bisect_utils.CheckRunGit(cmd) | 1463 return_code = bisect_utils.CheckRunGit(cmd) |
| 1062 os.chdir(cwd) | 1464 os.chdir(cwd) |
| (...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1178 if os.path.exists(os.path.join(abs_build_dir, 'out', build_type)): | 1580 if os.path.exists(os.path.join(abs_build_dir, 'out', build_type)): |
| 1179 output_dir = os.path.join(abs_build_dir, 'out', build_type) | 1581 output_dir = os.path.join(abs_build_dir, 'out', build_type) |
| 1180 else: | 1582 else: |
| 1181 raise IOError('Missing extracted folder %s ' % output_dir) | 1583 raise IOError('Missing extracted folder %s ' % output_dir) |
| 1182 | 1584 |
| 1183 print 'Moving build from %s to %s' % ( | 1585 print 'Moving build from %s to %s' % ( |
| 1184 output_dir, target_build_output_dir) | 1586 output_dir, target_build_output_dir) |
| 1185 shutil.move(output_dir, target_build_output_dir) | 1587 shutil.move(output_dir, target_build_output_dir) |
| 1186 return True | 1588 return True |
| 1187 except Exception as e: | 1589 except Exception as e: |
| 1188 print 'Somewthing went wrong while extracting archive file: %s' % e | 1590 print 'Something went wrong while extracting archive file: %s' % e |
| 1189 self.BackupOrRestoreOutputdirectory(restore=True) | 1591 self.BackupOrRestoreOutputdirectory(restore=True) |
| 1190 # Cleanup any leftovers from unzipping. | 1592 # Cleanup any leftovers from unzipping. |
| 1191 if os.path.exists(output_dir): | 1593 if os.path.exists(output_dir): |
| 1192 RmTreeAndMkDir(output_dir, skip_makedir=True) | 1594 RmTreeAndMkDir(output_dir, skip_makedir=True) |
| 1193 finally: | 1595 finally: |
| 1194 # Delete downloaded archive | 1596 # Delete downloaded archive |
| 1195 if os.path.exists(downloaded_file): | 1597 if os.path.exists(downloaded_file): |
| 1196 os.remove(downloaded_file) | 1598 os.remove(downloaded_file) |
| 1197 return False | 1599 return False |
| 1198 | 1600 |
| 1199 def WaitUntilBuildIsReady(self, fetch_build, bot_name, builder_host, | |
| 1200 builder_port, build_request_id, max_timeout): | |
| 1201 """Waits until build is produced by bisect builder on tryserver. | |
| 1202 | |
| 1203 Args: | |
| 1204 fetch_build: Function to check and download build from cloud storage. | |
| 1205 bot_name: Builder bot name on tryserver. | |
| 1206 builder_host Tryserver hostname. | |
| 1207 builder_port: Tryserver port. | |
| 1208 build_request_id: A unique ID of the build request posted to tryserver. | |
| 1209 max_timeout: Maximum time to wait for the build. | |
| 1210 | |
| 1211 Returns: | |
| 1212 Downloaded archive file path if exists, otherwise None. | |
| 1213 """ | |
| 1214 # Build number on the tryserver. | |
| 1215 build_num = None | |
| 1216 # Interval to check build on cloud storage. | |
| 1217 poll_interval = 60 | |
| 1218 # Interval to check build status on tryserver. | |
| 1219 status_check_interval = 600 | |
| 1220 last_status_check = time.time() | |
| 1221 start_time = time.time() | |
| 1222 while True: | |
| 1223 # Checks for build on gs://chrome-perf and download if exists. | |
| 1224 res = fetch_build() | |
| 1225 if res: | |
| 1226 return (res, 'Build successfully found') | |
| 1227 elapsed_status_check = time.time() - last_status_check | |
| 1228 # To avoid overloading tryserver with status check requests, we check | |
| 1229 # build status for every 10 mins. | |
| 1230 if elapsed_status_check > status_check_interval: | |
| 1231 last_status_check = time.time() | |
| 1232 if not build_num: | |
| 1233 # Get the build number on tryserver for the current build. | |
| 1234 build_num = bisect_builder.GetBuildNumFromBuilder( | |
| 1235 build_request_id, bot_name, builder_host, builder_port) | |
| 1236 # Check the status of build using the build number. | |
| 1237 # Note: Build is treated as PENDING if build number is not found | |
| 1238 # on the the tryserver. | |
| 1239 build_status, status_link = bisect_builder.GetBuildStatus( | |
| 1240 build_num, bot_name, builder_host, builder_port) | |
| 1241 if build_status == bisect_builder.FAILED: | |
| 1242 return (None, 'Failed to produce build, log: %s' % status_link) | |
| 1243 elapsed_time = time.time() - start_time | |
| 1244 if elapsed_time > max_timeout: | |
| 1245 return (None, 'Timed out: %ss without build' % max_timeout) | |
| 1246 | |
| 1247 print 'Time elapsed: %ss without build.' % elapsed_time | |
| 1248 time.sleep(poll_interval) | |
| 1249 # For some reason, mac bisect bots were not flushing stdout periodically. | |
| 1250 # As a result buildbot command is timed-out. Flush stdout on all platforms | |
| 1251 # while waiting for build. | |
| 1252 sys.stdout.flush() | |
| 1253 | |
| 1254 def PostBuildRequestAndWait(self, revision, fetch_build, patch=None): | 1601 def PostBuildRequestAndWait(self, revision, fetch_build, patch=None): |
| 1255 """POSTs the build request job to the tryserver instance. | 1602 """POSTs the build request job to the tryserver instance. |
| 1256 | 1603 |
| 1257 A try job build request is posted to tryserver.chromium.perf master, | 1604 A try job build request is posted to tryserver.chromium.perf master, |
| 1258 and waits for the binaries to be produced and archived on cloud storage. | 1605 and waits for the binaries to be produced and archived on cloud storage. |
| 1259 Once the build is ready and stored onto cloud, build archive is downloaded | 1606 Once the build is ready and stored onto cloud, build archive is downloaded |
| 1260 into the output folder. | 1607 into the output folder. |
| 1261 | 1608 |
| 1262 Args: | 1609 Args: |
| 1263 revision: A Git hash revision. | 1610 revision: A Git hash revision. |
| 1264 fetch_build: Function to check and download build from cloud storage. | 1611 fetch_build: Function to check and download build from cloud storage. |
| 1265 patch: A DEPS patch (used while bisecting 3rd party repositories). | 1612 patch: A DEPS patch (used while bisecting 3rd party repositories). |
| 1266 | 1613 |
| 1267 Returns: | 1614 Returns: |
| 1268 Downloaded archive file path when requested build exists and download is | 1615 Downloaded archive file path when requested build exists and download is |
| 1269 successful, otherwise None. | 1616 successful, otherwise None. |
| 1270 """ | 1617 """ |
| 1271 # Get SVN revision for the given SHA. | 1618 # Get SVN revision for the given SHA. |
| 1272 svn_revision = self.source_control.SVNFindRev(revision) | 1619 svn_revision = self.source_control.SVNFindRev(revision) |
| 1273 if not svn_revision: | 1620 if not svn_revision: |
| 1274 raise RuntimeError( | 1621 raise RuntimeError( |
| 1275 'Failed to determine SVN revision for %s' % revision) | 1622 'Failed to determine SVN revision for %s' % revision) |
| 1276 | 1623 |
| 1277 def GetBuilderNameAndBuildTime(target_platform, target_arch='ia32'): | 1624 def GetBuilderNameAndBuildTime(target_platform, target_arch='ia32'): |
| 1278 """Gets builder bot name and buildtime in seconds based on platform.""" | 1625 """Gets builder bot name and build time in seconds based on platform.""" |
| 1279 # Bot names should match the one listed in tryserver.chromium's | 1626 # Bot names should match the one listed in tryserver.chromium's |
| 1280 # master.cfg which produces builds for bisect. | 1627 # master.cfg which produces builds for bisect. |
| 1281 if bisect_utils.IsWindowsHost(): | 1628 if bisect_utils.IsWindowsHost(): |
| 1282 if bisect_utils.Is64BitWindows() and target_arch == 'x64': | 1629 if bisect_utils.Is64BitWindows() and target_arch == 'x64': |
| 1283 return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME) | 1630 return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME) |
| 1284 return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME) | 1631 return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME) |
| 1285 if bisect_utils.IsLinuxHost(): | 1632 if bisect_utils.IsLinuxHost(): |
| 1286 if target_platform == 'android': | 1633 if target_platform == 'android': |
| 1287 return ('android_perf_bisect_builder', MAX_LINUX_BUILD_TIME) | 1634 return ('android_perf_bisect_builder', MAX_LINUX_BUILD_TIME) |
| 1288 return ('linux_perf_bisect_builder', MAX_LINUX_BUILD_TIME) | 1635 return ('linux_perf_bisect_builder', MAX_LINUX_BUILD_TIME) |
| 1289 if bisect_utils.IsMacHost(): | 1636 if bisect_utils.IsMacHost(): |
| 1290 return ('mac_perf_bisect_builder', MAX_MAC_BUILD_TIME) | 1637 return ('mac_perf_bisect_builder', MAX_MAC_BUILD_TIME) |
| 1291 raise NotImplementedError('Unsupported Platform "%s".' % sys.platform) | 1638 raise NotImplementedError('Unsupported Platform "%s".' % sys.platform) |
| 1292 if not fetch_build: | 1639 if not fetch_build: |
| 1293 return False | 1640 return False |
| 1294 | 1641 |
| 1295 bot_name, build_timeout = GetBuilderNameAndBuildTime( | 1642 bot_name, build_timeout = GetBuilderNameAndBuildTime( |
| 1296 self.opts.target_platform, self.opts.target_arch) | 1643 self.opts.target_platform, self.opts.target_arch) |
| 1297 builder_host = self.opts.builder_host | 1644 builder_host = self.opts.builder_host |
| 1298 builder_port = self.opts.builder_port | 1645 builder_port = self.opts.builder_port |
| 1299 # Create a unique ID for each build request posted to tryserver builders. | 1646 # Create a unique ID for each build request posted to tryserver builders. |
| 1300 # This ID is added to "Reason" property in build's json. | 1647 # This ID is added to "Reason" property in build's json. |
| 1301 build_request_id = GetSHA1HexDigest( | 1648 build_request_id = GetSHA1HexDigest( |
| 1302 '%s-%s-%s' % (svn_revision, patch, time.time())) | 1649 '%s-%s-%s' % (svn_revision, patch, time.time())) |
| 1303 | 1650 |
| 1304 # Creates a try job description. | 1651 # Creates a try job description. |
| 1305 job_args = {'host': builder_host, | 1652 job_args = { |
| 1306 'port': builder_port, | 1653 'host': builder_host, |
| 1307 'revision': 'src@%s' % svn_revision, | 1654 'port': builder_port, |
| 1308 'bot': bot_name, | 1655 'revision': 'src@%s' % svn_revision, |
| 1309 'name': build_request_id | 1656 'bot': bot_name, |
| 1310 } | 1657 'name': build_request_id, |
| 1658 } |
| 1311 # Update patch information if supplied. | 1659 # Update patch information if supplied. |
| 1312 if patch: | 1660 if patch: |
| 1313 job_args['patch'] = patch | 1661 job_args['patch'] = patch |
| 1314 # Posts job to build the revision on the server. | 1662 # Posts job to build the revision on the server. |
| 1315 if bisect_builder.PostTryJob(job_args): | 1663 if bisect_builder.PostTryJob(job_args): |
| 1316 target_file, error_msg = self.WaitUntilBuildIsReady(fetch_build, | 1664 target_file, error_msg = _WaitUntilBuildIsReady( |
| 1317 bot_name, | 1665 fetch_build, bot_name, builder_host, builder_port, build_request_id, |
| 1318 builder_host, | 1666 build_timeout) |
| 1319 builder_port, | |
| 1320 build_request_id, | |
| 1321 build_timeout) | |
| 1322 if not target_file: | 1667 if not target_file: |
| 1323 print '%s [revision: %s]' % (error_msg, svn_revision) | 1668 print '%s [revision: %s]' % (error_msg, svn_revision) |
| 1324 return None | 1669 return None |
| 1325 return target_file | 1670 return target_file |
| 1326 print 'Failed to post build request for revision: [%s]' % svn_revision | 1671 print 'Failed to post build request for revision: [%s]' % svn_revision |
| 1327 return None | 1672 return None |
| 1328 | 1673 |
| 1329 def IsDownloadable(self, depot): | 1674 def IsDownloadable(self, depot): |
| 1330 """Checks if build is downloadable based on target platform and depot.""" | 1675 """Checks if build is downloadable based on target platform and depot.""" |
| 1331 if (self.opts.target_platform in ['chromium', 'android'] and | 1676 if (self.opts.target_platform in ['chromium', 'android'] and |
| (...skipping 20 matching lines...) Expand all Loading... |
| 1352 """ | 1697 """ |
| 1353 if not os.path.exists(deps_file): | 1698 if not os.path.exists(deps_file): |
| 1354 return False | 1699 return False |
| 1355 | 1700 |
| 1356 deps_var = DEPOT_DEPS_NAME[depot]['deps_var'] | 1701 deps_var = DEPOT_DEPS_NAME[depot]['deps_var'] |
| 1357 # Don't update DEPS file if deps_var is not set in DEPOT_DEPS_NAME. | 1702 # Don't update DEPS file if deps_var is not set in DEPOT_DEPS_NAME. |
| 1358 if not deps_var: | 1703 if not deps_var: |
| 1359 print 'DEPS update not supported for Depot: %s', depot | 1704 print 'DEPS update not supported for Depot: %s', depot |
| 1360 return False | 1705 return False |
| 1361 | 1706 |
| 1362 # Hack to Angle repository because, in DEPS file "vars" dictionary variable | 1707 # Hack for Angle repository. In the DEPS file, "vars" dictionary variable |
| 1363 # contains "angle_revision" key that holds git hash instead of SVN revision. | 1708 # contains "angle_revision" key that holds git hash instead of SVN revision. |
| 1364 # And sometime "angle_revision" key is not specified in "vars" variable, | 1709 # And sometime "angle_revision" key is not specified in "vars" variable. |
| 1365 # in such cases check "deps" dictionary variable that matches | 1710 # In such cases check, "deps" dictionary variable that matches |
| 1366 # angle.git@[a-fA-F0-9]{40}$ and replace git hash. | 1711 # angle.git@[a-fA-F0-9]{40}$ and replace git hash. |
| 1367 if depot == 'angle': | 1712 if depot == 'angle': |
| 1368 return self.UpdateDEPSForAngle(revision, depot, deps_file) | 1713 return _UpdateDEPSForAngle(revision, depot, deps_file) |
| 1369 | 1714 |
| 1370 try: | 1715 try: |
| 1371 deps_contents = ReadStringFromFile(deps_file) | 1716 deps_contents = ReadStringFromFile(deps_file) |
| 1372 # Check whether the depot and revision pattern in DEPS file vars | 1717 # Check whether the depot and revision pattern in DEPS file vars |
| 1373 # e.g. for webkit the format is "webkit_revision": "12345". | 1718 # e.g. for webkit the format is "webkit_revision": "12345". |
| 1374 deps_revision = re.compile(r'(?<="%s": ")([0-9]+)(?=")' % deps_var, | 1719 deps_revision = re.compile(r'(?<="%s": ")([0-9]+)(?=")' % deps_var, |
| 1375 re.MULTILINE) | 1720 re.MULTILINE) |
| 1376 match = re.search(deps_revision, deps_contents) | 1721 match = re.search(deps_revision, deps_contents) |
| 1377 if match: | 1722 if match: |
| 1378 svn_revision = self.source_control.SVNFindRev( | 1723 svn_revision = self.source_control.SVNFindRev( |
| 1379 revision, self._GetDepotDirectory(depot)) | 1724 revision, self._GetDepotDirectory(depot)) |
| 1380 if not svn_revision: | 1725 if not svn_revision: |
| 1381 print 'Could not determine SVN revision for %s' % revision | 1726 print 'Could not determine SVN revision for %s' % revision |
| 1382 return False | 1727 return False |
| 1383 # Update the revision information for the given depot | 1728 # Update the revision information for the given depot |
| 1384 new_data = re.sub(deps_revision, str(svn_revision), deps_contents) | 1729 new_data = re.sub(deps_revision, str(svn_revision), deps_contents) |
| 1385 | 1730 |
| 1386 # For v8_bleeding_edge revisions change V8 branch in order | 1731 # For v8_bleeding_edge revisions change V8 branch in order |
| 1387 # to fetch bleeding edge revision. | 1732 # to fetch bleeding edge revision. |
| 1388 if depot == 'v8_bleeding_edge': | 1733 if depot == 'v8_bleeding_edge': |
| 1389 new_data = self.UpdateV8Branch(new_data) | 1734 new_data = _UpdateV8Branch(new_data) |
| 1390 if not new_data: | 1735 if not new_data: |
| 1391 return False | 1736 return False |
| 1392 # Write changes to DEPS file | 1737 # Write changes to DEPS file |
| 1393 WriteStringToFile(new_data, deps_file) | 1738 WriteStringToFile(new_data, deps_file) |
| 1394 return True | 1739 return True |
| 1395 except IOError, e: | 1740 except IOError, e: |
| 1396 print 'Something went wrong while updating DEPS file. [%s]' % e | 1741 print 'Something went wrong while updating DEPS file. [%s]' % e |
| 1397 return False | 1742 return False |
| 1398 | 1743 |
| 1399 def UpdateV8Branch(self, deps_content): | |
| 1400 """Updates V8 branch in DEPS file to process v8_bleeding_edge. | |
| 1401 | |
| 1402 Check for "v8_branch" in DEPS file if exists update its value | |
| 1403 with v8_bleeding_edge branch. Note: "v8_branch" is added to DEPS | |
| 1404 variable from DEPS revision 254916, therefore check for "src/v8": | |
| 1405 <v8 source path> in DEPS in order to support prior DEPS revisions | |
| 1406 and update it. | |
| 1407 | |
| 1408 Args: | |
| 1409 deps_content: DEPS file contents to be modified. | |
| 1410 | |
| 1411 Returns: | |
| 1412 Modified DEPS file contents as a string. | |
| 1413 """ | |
| 1414 new_branch = r'branches/bleeding_edge' | |
| 1415 v8_branch_pattern = re.compile(r'(?<="v8_branch": ")(.*)(?=")') | |
| 1416 if re.search(v8_branch_pattern, deps_content): | |
| 1417 deps_content = re.sub(v8_branch_pattern, new_branch, deps_content) | |
| 1418 else: | |
| 1419 # Replaces the branch assigned to "src/v8" key in DEPS file. | |
| 1420 # Format of "src/v8" in DEPS: | |
| 1421 # "src/v8": | |
| 1422 # (Var("googlecode_url") % "v8") + "/trunk@" + Var("v8_revision"), | |
| 1423 # So, "/trunk@" is replace with "/branches/bleeding_edge@" | |
| 1424 v8_src_pattern = re.compile( | |
| 1425 r'(?<="v8"\) \+ "/)(.*)(?=@" \+ Var\("v8_revision"\))', re.MULTILINE) | |
| 1426 if re.search(v8_src_pattern, deps_content): | |
| 1427 deps_content = re.sub(v8_src_pattern, new_branch, deps_content) | |
| 1428 return deps_content | |
| 1429 | |
| 1430 def UpdateDEPSForAngle(self, revision, depot, deps_file): | |
| 1431 """Updates DEPS file with new revision for Angle repository. | |
| 1432 | |
| 1433 This is a hack for Angle depot case because, in DEPS file "vars" dictionary | |
| 1434 variable contains "angle_revision" key that holds git hash instead of | |
| 1435 SVN revision. | |
| 1436 | |
| 1437 And sometimes "angle_revision" key is not specified in "vars" variable, | |
| 1438 in such cases check "deps" dictionary variable that matches | |
| 1439 angle.git@[a-fA-F0-9]{40}$ and replace git hash. | |
| 1440 """ | |
| 1441 deps_var = DEPOT_DEPS_NAME[depot]['deps_var'] | |
| 1442 try: | |
| 1443 deps_contents = ReadStringFromFile(deps_file) | |
| 1444 # Check whether the depot and revision pattern in DEPS file vars variable | |
| 1445 # e.g. "angle_revision": "fa63e947cb3eccf463648d21a05d5002c9b8adfa". | |
| 1446 angle_rev_pattern = re.compile(r'(?<="%s": ")([a-fA-F0-9]{40})(?=")' % | |
| 1447 deps_var, re.MULTILINE) | |
| 1448 match = re.search(angle_rev_pattern % deps_var, deps_contents) | |
| 1449 if match: | |
| 1450 # Update the revision information for the given depot | |
| 1451 new_data = re.sub(angle_rev_pattern, revision, deps_contents) | |
| 1452 else: | |
| 1453 # Check whether the depot and revision pattern in DEPS file deps | |
| 1454 # variable. e.g., | |
| 1455 # "src/third_party/angle": Var("chromium_git") + | |
| 1456 # "/angle/angle.git@fa63e947cb3eccf463648d21a05d5002c9b8adfa",. | |
| 1457 angle_rev_pattern = re.compile( | |
| 1458 r'(?<=angle\.git@)([a-fA-F0-9]{40})(?=")', re.MULTILINE) | |
| 1459 match = re.search(angle_rev_pattern, deps_contents) | |
| 1460 if not match: | |
| 1461 print 'Could not find angle revision information in DEPS file.' | |
| 1462 return False | |
| 1463 new_data = re.sub(angle_rev_pattern, revision, deps_contents) | |
| 1464 # Write changes to DEPS file | |
| 1465 WriteStringToFile(new_data, deps_file) | |
| 1466 return True | |
| 1467 except IOError, e: | |
| 1468 print 'Something went wrong while updating DEPS file, %s' % e | |
| 1469 return False | |
| 1470 | |
| 1471 def CreateDEPSPatch(self, depot, revision): | 1744 def CreateDEPSPatch(self, depot, revision): |
| 1472 """Modifies DEPS and returns diff as text. | 1745 """Modifies DEPS and returns diff as text. |
| 1473 | 1746 |
| 1474 Args: | 1747 Args: |
| 1475 depot: Current depot being bisected. | 1748 depot: Current depot being bisected. |
| 1476 revision: A git hash revision of the dependency repository. | 1749 revision: A git hash revision of the dependency repository. |
| 1477 | 1750 |
| 1478 Returns: | 1751 Returns: |
| 1479 A tuple with git hash of chromium revision and DEPS patch text. | 1752 A tuple with git hash of chromium revision and DEPS patch text. |
| 1480 """ | 1753 """ |
| 1481 deps_file_path = os.path.join(self.src_cwd, bisect_utils.FILE_DEPS) | 1754 deps_file_path = os.path.join(self.src_cwd, bisect_utils.FILE_DEPS) |
| 1482 if not os.path.exists(deps_file_path): | 1755 if not os.path.exists(deps_file_path): |
| 1483 raise RuntimeError('DEPS file does not exists.[%s]' % deps_file_path) | 1756 raise RuntimeError('DEPS file does not exists.[%s]' % deps_file_path) |
| 1484 # Get current chromium revision (git hash). | 1757 # Get current chromium revision (git hash). |
| 1485 cmd = ['rev-parse', 'HEAD'] | 1758 cmd = ['rev-parse', 'HEAD'] |
| 1486 chromium_sha = bisect_utils.CheckRunGit(cmd).strip() | 1759 chromium_sha = bisect_utils.CheckRunGit(cmd).strip() |
| 1487 if not chromium_sha: | 1760 if not chromium_sha: |
| 1488 raise RuntimeError('Failed to determine Chromium revision for %s' % | 1761 raise RuntimeError('Failed to determine Chromium revision for %s' % |
| 1489 revision) | 1762 revision) |
| 1490 if ('chromium' in DEPOT_DEPS_NAME[depot]['from'] or | 1763 if ('chromium' in DEPOT_DEPS_NAME[depot]['from'] or |
| 1491 'v8' in DEPOT_DEPS_NAME[depot]['from']): | 1764 'v8' in DEPOT_DEPS_NAME[depot]['from']): |
| 1492 # Checkout DEPS file for the current chromium revision. | 1765 # Checkout DEPS file for the current chromium revision. |
| 1493 if self.source_control.CheckoutFileAtRevision(bisect_utils.FILE_DEPS, | 1766 if self.source_control.CheckoutFileAtRevision( |
| 1494 chromium_sha, | 1767 bisect_utils.FILE_DEPS, chromium_sha, cwd=self.src_cwd): |
| 1495 cwd=self.src_cwd): | |
| 1496 if self.UpdateDeps(revision, depot, deps_file_path): | 1768 if self.UpdateDeps(revision, depot, deps_file_path): |
| 1497 diff_command = ['diff', | 1769 diff_command = [ |
| 1498 '--src-prefix=src/', | 1770 'diff', |
| 1499 '--dst-prefix=src/', | 1771 '--src-prefix=src/', |
| 1500 '--no-ext-diff', | 1772 '--dst-prefix=src/', |
| 1501 bisect_utils.FILE_DEPS] | 1773 '--no-ext-diff', |
| 1502 diff_text = bisect_utils.CheckRunGit( | 1774 bisect_utils.FILE_DEPS, |
| 1503 diff_command, cwd=self.src_cwd) | 1775 ] |
| 1776 diff_text = bisect_utils.CheckRunGit(diff_command, cwd=self.src_cwd) |
| 1504 return (chromium_sha, ChangeBackslashToSlashInPatch(diff_text)) | 1777 return (chromium_sha, ChangeBackslashToSlashInPatch(diff_text)) |
| 1505 else: | 1778 else: |
| 1506 raise RuntimeError('Failed to update DEPS file for chromium: [%s]' % | 1779 raise RuntimeError( |
| 1507 chromium_sha) | 1780 'Failed to update DEPS file for chromium: [%s]' % chromium_sha) |
| 1508 else: | 1781 else: |
| 1509 raise RuntimeError('DEPS checkout Failed for chromium revision : [%s]' % | 1782 raise RuntimeError( |
| 1510 chromium_sha) | 1783 'DEPS checkout Failed for chromium revision : [%s]' % chromium_sha) |
| 1511 return (None, None) | 1784 return (None, None) |
| 1512 | 1785 |
| 1513 def BuildCurrentRevision(self, depot, revision=None): | 1786 def BuildCurrentRevision(self, depot, revision=None): |
| 1514 """Builds chrome and performance_ui_tests on the current revision. | 1787 """Builds chrome and performance_ui_tests on the current revision. |
| 1515 | 1788 |
| 1516 Returns: | 1789 Returns: |
| 1517 True if the build was successful. | 1790 True if the build was successful. |
| 1518 """ | 1791 """ |
| 1519 if self.opts.debug_ignore_build: | 1792 if self.opts.debug_ignore_build: |
| 1520 return True | 1793 return True |
| 1521 cwd = os.getcwd() | 1794 cwd = os.getcwd() |
| 1522 os.chdir(self.src_cwd) | 1795 os.chdir(self.src_cwd) |
| 1523 # Fetch build archive for the given revision from the cloud storage when | 1796 # Fetch build archive for the given revision from the cloud storage when |
| 1524 # the storage bucket is passed. | 1797 # the storage bucket is passed. |
| 1525 if self.IsDownloadable(depot) and revision: | 1798 if self.IsDownloadable(depot) and revision: |
| 1526 deps_patch = None | 1799 deps_patch = None |
| 1527 if depot != 'chromium': | 1800 if depot != 'chromium': |
| 1528 # Create a DEPS patch with new revision for dependency repository. | 1801 # Create a DEPS patch with new revision for dependency repository. |
| 1529 (revision, deps_patch) = self.CreateDEPSPatch(depot, revision) | 1802 revision, deps_patch = self.CreateDEPSPatch(depot, revision) |
| 1530 if self.DownloadCurrentBuild(revision, patch=deps_patch): | 1803 if self.DownloadCurrentBuild(revision, patch=deps_patch): |
| 1531 os.chdir(cwd) | 1804 os.chdir(cwd) |
| 1532 if deps_patch: | 1805 if deps_patch: |
| 1533 # Reverts the changes to DEPS file. | 1806 # Reverts the changes to DEPS file. |
| 1534 self.source_control.CheckoutFileAtRevision(bisect_utils.FILE_DEPS, | 1807 self.source_control.CheckoutFileAtRevision( |
| 1535 revision, | 1808 bisect_utils.FILE_DEPS, revision, cwd=self.src_cwd) |
| 1536 cwd=self.src_cwd) | |
| 1537 return True | 1809 return True |
| 1538 return False | 1810 return False |
| 1539 | 1811 |
| 1540 # These codes are executed when bisect bots builds binaries locally. | 1812 # These codes are executed when bisect bots builds binaries locally. |
| 1541 build_success = self.builder.Build(depot, self.opts) | 1813 build_success = self.builder.Build(depot, self.opts) |
| 1542 os.chdir(cwd) | 1814 os.chdir(cwd) |
| 1543 return build_success | 1815 return build_success |
| 1544 | 1816 |
| 1545 def RunGClientHooks(self): | 1817 def RunGClientHooks(self): |
| 1546 """Runs gclient with runhooks command. | 1818 """Runs gclient with runhooks command. |
| 1547 | 1819 |
| 1548 Returns: | 1820 Returns: |
| 1549 True if gclient reports no errors. | 1821 True if gclient reports no errors. |
| 1550 """ | 1822 """ |
| 1551 | |
| 1552 if self.opts.debug_ignore_build: | 1823 if self.opts.debug_ignore_build: |
| 1553 return True | 1824 return True |
| 1554 | |
| 1555 return not bisect_utils.RunGClient(['runhooks'], cwd=self.src_cwd) | 1825 return not bisect_utils.RunGClient(['runhooks'], cwd=self.src_cwd) |
| 1556 | 1826 |
| 1557 def TryParseHistogramValuesFromOutput(self, metric, text): | |
| 1558 """Attempts to parse a metric in the format HISTOGRAM <graph: <trace>. | |
| 1559 | |
| 1560 Args: | |
| 1561 metric: The metric as a list of [<trace>, <value>] strings. | |
| 1562 text: The text to parse the metric values from. | |
| 1563 | |
| 1564 Returns: | |
| 1565 A list of floating point numbers found. | |
| 1566 """ | |
| 1567 metric_formatted = 'HISTOGRAM %s: %s= ' % (metric[0], metric[1]) | |
| 1568 | |
| 1569 text_lines = text.split('\n') | |
| 1570 values_list = [] | |
| 1571 | |
| 1572 for current_line in text_lines: | |
| 1573 if metric_formatted in current_line: | |
| 1574 current_line = current_line[len(metric_formatted):] | |
| 1575 | |
| 1576 try: | |
| 1577 histogram_values = eval(current_line) | |
| 1578 | |
| 1579 for b in histogram_values['buckets']: | |
| 1580 average_for_bucket = float(b['high'] + b['low']) * 0.5 | |
| 1581 # Extends the list with N-elements with the average for that bucket. | |
| 1582 values_list.extend([average_for_bucket] * b['count']) | |
| 1583 except: | |
| 1584 pass | |
| 1585 | |
| 1586 return values_list | |
| 1587 | |
| 1588 def TryParseResultValuesFromOutput(self, metric, text): | |
| 1589 """Attempts to parse a metric in the format RESULT <graph>: <trace>= ... | |
| 1590 | |
| 1591 Args: | |
| 1592 metric: The metric as a list of [<trace>, <value>] strings. | |
| 1593 text: The text to parse the metric values from. | |
| 1594 | |
| 1595 Returns: | |
| 1596 A list of floating point numbers found. | |
| 1597 """ | |
| 1598 # Format is: RESULT <graph>: <trace>= <value> <units> | |
| 1599 metric_re = re.escape('RESULT %s: %s=' % (metric[0], metric[1])) | |
| 1600 | |
| 1601 # The log will be parsed looking for format: | |
| 1602 # <*>RESULT <graph_name>: <trace_name>= <value> | |
| 1603 single_result_re = re.compile( | |
| 1604 metric_re + '\s*(?P<VALUE>[-]?\d*(\.\d*)?)') | |
| 1605 | |
| 1606 # The log will be parsed looking for format: | |
| 1607 # <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...] | |
| 1608 multi_results_re = re.compile( | |
| 1609 metric_re + '\s*\[\s*(?P<VALUES>[-]?[\d\., ]+)\s*\]') | |
| 1610 | |
| 1611 # The log will be parsed looking for format: | |
| 1612 # <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} | |
| 1613 mean_stddev_re = re.compile( | |
| 1614 metric_re + | |
| 1615 '\s*\{\s*(?P<MEAN>[-]?\d*(\.\d*)?),\s*(?P<STDDEV>\d+(\.\d*)?)\s*\}') | |
| 1616 | |
| 1617 text_lines = text.split('\n') | |
| 1618 values_list = [] | |
| 1619 for current_line in text_lines: | |
| 1620 # Parse the output from the performance test for the metric we're | |
| 1621 # interested in. | |
| 1622 single_result_match = single_result_re.search(current_line) | |
| 1623 multi_results_match = multi_results_re.search(current_line) | |
| 1624 mean_stddev_match = mean_stddev_re.search(current_line) | |
| 1625 if (not single_result_match is None and | |
| 1626 single_result_match.group('VALUE')): | |
| 1627 values_list += [single_result_match.group('VALUE')] | |
| 1628 elif (not multi_results_match is None and | |
| 1629 multi_results_match.group('VALUES')): | |
| 1630 metric_values = multi_results_match.group('VALUES') | |
| 1631 values_list += metric_values.split(',') | |
| 1632 elif (not mean_stddev_match is None and | |
| 1633 mean_stddev_match.group('MEAN')): | |
| 1634 values_list += [mean_stddev_match.group('MEAN')] | |
| 1635 | |
| 1636 values_list = [float(v) for v in values_list | |
| 1637 if bisect_utils.IsStringFloat(v)] | |
| 1638 | |
| 1639 # If the metric is times/t, we need to sum the timings in order to get | |
| 1640 # similar regression results as the try-bots. | |
| 1641 metrics_to_sum = [['times', 't'], ['times', 'page_load_time'], | |
| 1642 ['cold_times', 'page_load_time'], ['warm_times', 'page_load_time']] | |
| 1643 | |
| 1644 if metric in metrics_to_sum: | |
| 1645 if values_list: | |
| 1646 values_list = [reduce(lambda x, y: float(x) + float(y), values_list)] | |
| 1647 | |
| 1648 return values_list | |
| 1649 | |
| 1650 def ParseMetricValuesFromOutput(self, metric, text): | |
| 1651 """Parses output from performance_ui_tests and retrieves the results for | |
| 1652 a given metric. | |
| 1653 | |
| 1654 Args: | |
| 1655 metric: The metric as a list of [<trace>, <value>] strings. | |
| 1656 text: The text to parse the metric values from. | |
| 1657 | |
| 1658 Returns: | |
| 1659 A list of floating point numbers found. | |
| 1660 """ | |
| 1661 metric_values = self.TryParseResultValuesFromOutput(metric, text) | |
| 1662 | |
| 1663 if not metric_values: | |
| 1664 metric_values = self.TryParseHistogramValuesFromOutput(metric, text) | |
| 1665 | |
| 1666 return metric_values | |
| 1667 | |
| 1668 def _GenerateProfileIfNecessary(self, command_args): | |
| 1669 """Checks the command line of the performance test for dependencies on | |
| 1670 profile generation, and runs tools/perf/generate_profile as necessary. | |
| 1671 | |
| 1672 Args: | |
| 1673 command_args: Command line being passed to performance test, as a list. | |
| 1674 | |
| 1675 Returns: | |
| 1676 False if profile generation was necessary and failed, otherwise True. | |
| 1677 """ | |
| 1678 | |
| 1679 if '--profile-dir' in ' '.join(command_args): | |
| 1680 # If we were using python 2.7+, we could just use the argparse | |
| 1681 # module's parse_known_args to grab --profile-dir. Since some of the | |
| 1682 # bots still run 2.6, have to grab the arguments manually. | |
| 1683 arg_dict = {} | |
| 1684 args_to_parse = ['--profile-dir', '--browser'] | |
| 1685 | |
| 1686 for arg_to_parse in args_to_parse: | |
| 1687 for i, current_arg in enumerate(command_args): | |
| 1688 if arg_to_parse in current_arg: | |
| 1689 current_arg_split = current_arg.split('=') | |
| 1690 | |
| 1691 # Check 2 cases, --arg=<val> and --arg <val> | |
| 1692 if len(current_arg_split) == 2: | |
| 1693 arg_dict[arg_to_parse] = current_arg_split[1] | |
| 1694 elif i + 1 < len(command_args): | |
| 1695 arg_dict[arg_to_parse] = command_args[i+1] | |
| 1696 | |
| 1697 path_to_generate = os.path.join('tools', 'perf', 'generate_profile') | |
| 1698 | |
| 1699 if arg_dict.has_key('--profile-dir') and arg_dict.has_key('--browser'): | |
| 1700 profile_path, profile_type = os.path.split(arg_dict['--profile-dir']) | |
| 1701 return not bisect_utils.RunProcess(['python', path_to_generate, | |
| 1702 '--profile-type-to-generate', profile_type, | |
| 1703 '--browser', arg_dict['--browser'], '--output-dir', profile_path]) | |
| 1704 return False | |
| 1705 return True | |
| 1706 | |
| 1707 def _IsBisectModeUsingMetric(self): | 1827 def _IsBisectModeUsingMetric(self): |
| 1708 return self.opts.bisect_mode in [BISECT_MODE_MEAN, BISECT_MODE_STD_DEV] | 1828 return self.opts.bisect_mode in [BISECT_MODE_MEAN, BISECT_MODE_STD_DEV] |
| 1709 | 1829 |
| 1710 def _IsBisectModeReturnCode(self): | 1830 def _IsBisectModeReturnCode(self): |
| 1711 return self.opts.bisect_mode in [BISECT_MODE_RETURN_CODE] | 1831 return self.opts.bisect_mode in [BISECT_MODE_RETURN_CODE] |
| 1712 | 1832 |
| 1713 def _IsBisectModeStandardDeviation(self): | 1833 def _IsBisectModeStandardDeviation(self): |
| 1714 return self.opts.bisect_mode in [BISECT_MODE_STD_DEV] | 1834 return self.opts.bisect_mode in [BISECT_MODE_STD_DEV] |
| 1715 | 1835 |
| 1716 def GetCompatibleCommand(self, command_to_run, revision, depot): | 1836 def GetCompatibleCommand(self, command_to_run, revision, depot): |
| (...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1770 'std_dev': 0.0, | 1890 'std_dev': 0.0, |
| 1771 'values': [0.0] | 1891 'values': [0.0] |
| 1772 } | 1892 } |
| 1773 return (fake_results, success_code) | 1893 return (fake_results, success_code) |
| 1774 | 1894 |
| 1775 # For Windows platform set posix=False, to parse windows paths correctly. | 1895 # For Windows platform set posix=False, to parse windows paths correctly. |
| 1776 # On Windows, path separators '\' or '\\' are replace by '' when posix=True, | 1896 # On Windows, path separators '\' or '\\' are replace by '' when posix=True, |
| 1777 # refer to http://bugs.python.org/issue1724822. By default posix=True. | 1897 # refer to http://bugs.python.org/issue1724822. By default posix=True. |
| 1778 args = shlex.split(command_to_run, posix=not bisect_utils.IsWindowsHost()) | 1898 args = shlex.split(command_to_run, posix=not bisect_utils.IsWindowsHost()) |
| 1779 | 1899 |
| 1780 if not self._GenerateProfileIfNecessary(args): | 1900 if not _GenerateProfileIfNecessary(args): |
| 1781 err_text = 'Failed to generate profile for performance test.' | 1901 err_text = 'Failed to generate profile for performance test.' |
| 1782 return (err_text, failure_code) | 1902 return (err_text, failure_code) |
| 1783 | 1903 |
| 1784 # If running a Telemetry test for Chrome OS, insert the remote IP and | 1904 # If running a Telemetry test for Chrome OS, insert the remote IP and |
| 1785 # identity parameters. | 1905 # identity parameters. |
| 1786 is_telemetry = bisect_utils.IsTelemetryCommand(command_to_run) | 1906 is_telemetry = bisect_utils.IsTelemetryCommand(command_to_run) |
| 1787 if self.opts.target_platform == 'cros' and is_telemetry: | 1907 if self.opts.target_platform == 'cros' and is_telemetry: |
| 1788 args.append('--remote=%s' % self.opts.cros_remote_ip) | 1908 args.append('--remote=%s' % self.opts.cros_remote_ip) |
| 1789 args.append('--identity=%s' % CROS_TEST_KEY_PATH) | 1909 args.append('--identity=%s' % CROS_TEST_KEY_PATH) |
| 1790 | 1910 |
| 1791 start_time = time.time() | 1911 start_time = time.time() |
| 1792 | 1912 |
| 1793 metric_values = [] | 1913 metric_values = [] |
| 1794 output_of_all_runs = '' | 1914 output_of_all_runs = '' |
| 1795 for i in xrange(self.opts.repeat_test_count): | 1915 for i in xrange(self.opts.repeat_test_count): |
| 1796 # Can ignore the return code since if the tests fail, it won't return 0. | 1916 # Can ignore the return code since if the tests fail, it won't return 0. |
| 1797 current_args = copy.copy(args) | 1917 current_args = copy.copy(args) |
| 1798 if is_telemetry: | 1918 if is_telemetry: |
| 1799 if i == 0 and reset_on_first_run: | 1919 if i == 0 and reset_on_first_run: |
| 1800 current_args.append('--reset-results') | 1920 current_args.append('--reset-results') |
| 1801 elif i == self.opts.repeat_test_count - 1 and upload_on_last_run: | 1921 elif i == self.opts.repeat_test_count - 1 and upload_on_last_run: |
| 1802 current_args.append('--upload-results') | 1922 current_args.append('--upload-results') |
| 1803 if results_label: | 1923 if results_label: |
| 1804 current_args.append('--results-label=%s' % results_label) | 1924 current_args.append('--results-label=%s' % results_label) |
| 1805 try: | 1925 try: |
| 1806 (output, return_code) = bisect_utils.RunProcessAndRetrieveOutput( | 1926 output, return_code = bisect_utils.RunProcessAndRetrieveOutput( |
| 1807 current_args, cwd=self.src_cwd) | 1927 current_args, cwd=self.src_cwd) |
| 1808 except OSError, e: | 1928 except OSError, e: |
| 1809 if e.errno == errno.ENOENT: | 1929 if e.errno == errno.ENOENT: |
| 1810 err_text = ('Something went wrong running the performance test. ' | 1930 err_text = ('Something went wrong running the performance test. ' |
| 1811 'Please review the command line:\n\n') | 1931 'Please review the command line:\n\n') |
| 1812 if 'src/' in ' '.join(args): | 1932 if 'src/' in ' '.join(args): |
| 1813 err_text += ('Check that you haven\'t accidentally specified a ' | 1933 err_text += ('Check that you haven\'t accidentally specified a ' |
| 1814 'path with src/ in the command.\n\n') | 1934 'path with src/ in the command.\n\n') |
| 1815 err_text += ' '.join(args) | 1935 err_text += ' '.join(args) |
| 1816 err_text += '\n' | 1936 err_text += '\n' |
| 1817 | 1937 |
| 1818 return (err_text, failure_code) | 1938 return (err_text, failure_code) |
| 1819 raise | 1939 raise |
| 1820 | 1940 |
| 1821 output_of_all_runs += output | 1941 output_of_all_runs += output |
| 1822 if self.opts.output_buildbot_annotations: | 1942 if self.opts.output_buildbot_annotations: |
| 1823 print output | 1943 print output |
| 1824 | 1944 |
| 1825 if self._IsBisectModeUsingMetric(): | 1945 if self._IsBisectModeUsingMetric(): |
| 1826 metric_values += self.ParseMetricValuesFromOutput(metric, output) | 1946 metric_values += _ParseMetricValuesFromOutput(metric, output) |
| 1827 # If we're bisecting on a metric (ie, changes in the mean or | 1947 # If we're bisecting on a metric (ie, changes in the mean or |
| 1828 # standard deviation) and no metric values are produced, bail out. | 1948 # standard deviation) and no metric values are produced, bail out. |
| 1829 if not metric_values: | 1949 if not metric_values: |
| 1830 break | 1950 break |
| 1831 elif self._IsBisectModeReturnCode(): | 1951 elif self._IsBisectModeReturnCode(): |
| 1832 metric_values.append(return_code) | 1952 metric_values.append(return_code) |
| 1833 | 1953 |
| 1834 elapsed_minutes = (time.time() - start_time) / 60.0 | 1954 elapsed_minutes = (time.time() - start_time) / 60.0 |
| 1835 if elapsed_minutes >= self.opts.max_time_minutes: | 1955 if elapsed_minutes >= self.opts.max_time_minutes: |
| 1836 break | 1956 break |
| 1837 | 1957 |
| 1838 if len(metric_values) == 0: | 1958 if len(metric_values) == 0: |
| 1839 err_text = 'Metric %s was not found in the test output.' % metric | 1959 err_text = 'Metric %s was not found in the test output.' % metric |
| 1840 # TODO(qyearsley): Consider also getting and displaying a list of metrics | 1960 # TODO(qyearsley): Consider also getting and displaying a list of metrics |
| 1841 # that were found in the output here. | 1961 # that were found in the output here. |
| 1842 return (err_text, failure_code, output_of_all_runs) | 1962 return (err_text, failure_code, output_of_all_runs) |
| 1843 | 1963 |
| 1844 # If we're bisecting on return codes, we're really just looking for zero vs | 1964 # If we're bisecting on return codes, we're really just looking for zero vs |
| 1845 # non-zero. | 1965 # non-zero. |
| 1846 if self._IsBisectModeReturnCode(): | 1966 if self._IsBisectModeReturnCode(): |
| 1847 # If any of the return codes is non-zero, output 1. | 1967 # If any of the return codes is non-zero, output 1. |
| 1848 overall_return_code = 0 if ( | 1968 overall_return_code = 0 if ( |
| 1849 all(current_value == 0 for current_value in metric_values)) else 1 | 1969 all(current_value == 0 for current_value in metric_values)) else 1 |
| 1850 | 1970 |
| 1851 values = { | 1971 values = { |
| 1852 'mean': overall_return_code, | 1972 'mean': overall_return_code, |
| 1853 'std_err': 0.0, | 1973 'std_err': 0.0, |
| 1854 'std_dev': 0.0, | 1974 'std_dev': 0.0, |
| 1855 'values': metric_values, | 1975 'values': metric_values, |
| 1856 } | 1976 } |
| 1857 | 1977 |
| 1858 print 'Results of performance test: Command returned with %d' % ( | 1978 print 'Results of performance test: Command returned with %d' % ( |
| 1859 overall_return_code) | 1979 overall_return_code) |
| 1860 print | 1980 print |
| 1861 else: | 1981 else: |
| 1862 # Need to get the average value if there were multiple values. | 1982 # Need to get the average value if there were multiple values. |
| 1863 truncated_mean = math_utils.TruncatedMean( | 1983 truncated_mean = math_utils.TruncatedMean( |
| 1864 metric_values, self.opts.truncate_percent) | 1984 metric_values, self.opts.truncate_percent) |
| 1865 standard_err = math_utils.StandardError(metric_values) | 1985 standard_err = math_utils.StandardError(metric_values) |
| 1866 standard_dev = math_utils.StandardDeviation(metric_values) | 1986 standard_dev = math_utils.StandardDeviation(metric_values) |
| 1867 | 1987 |
| 1868 if self._IsBisectModeStandardDeviation(): | 1988 if self._IsBisectModeStandardDeviation(): |
| 1869 metric_values = [standard_dev] | 1989 metric_values = [standard_dev] |
| 1870 | 1990 |
| 1871 values = { | 1991 values = { |
| 1872 'mean': truncated_mean, | 1992 'mean': truncated_mean, |
| 1873 'std_err': standard_err, | 1993 'std_err': standard_err, |
| 1874 'std_dev': standard_dev, | 1994 'std_dev': standard_dev, |
| 1875 'values': metric_values, | 1995 'values': metric_values, |
| 1876 } | 1996 } |
| 1877 | 1997 |
| 1878 print 'Results of performance test: %12f %12f' % ( | 1998 print 'Results of performance test: %12f %12f' % ( |
| 1879 truncated_mean, standard_err) | 1999 truncated_mean, standard_err) |
| 1880 print | 2000 print |
| 1881 return (values, success_code, output_of_all_runs) | 2001 return (values, success_code, output_of_all_runs) |
| 1882 | 2002 |
| 1883 def FindAllRevisionsToSync(self, revision, depot): | 2003 def FindAllRevisionsToSync(self, revision, depot): |
| 1884 """Finds all dependant revisions and depots that need to be synced for a | 2004 """Finds all dependant revisions and depots that need to be synced for a |
| 1885 given revision. This is only useful in the git workflow, as an svn depot | 2005 given revision. This is only useful in the git workflow, as an svn depot |
| (...skipping 12 matching lines...) Expand all Loading... |
| 1898 """ | 2018 """ |
| 1899 revisions_to_sync = [[depot, revision]] | 2019 revisions_to_sync = [[depot, revision]] |
| 1900 | 2020 |
| 1901 is_base = ((depot == 'chromium') or (depot == 'cros') or | 2021 is_base = ((depot == 'chromium') or (depot == 'cros') or |
| 1902 (depot == 'android-chrome')) | 2022 (depot == 'android-chrome')) |
| 1903 | 2023 |
| 1904 # Some SVN depots were split into multiple git depots, so we need to | 2024 # Some SVN depots were split into multiple git depots, so we need to |
| 1905 # figure out for each mirror which git revision to grab. There's no | 2025 # figure out for each mirror which git revision to grab. There's no |
| 1906 # guarantee that the SVN revision will exist for each of the dependant | 2026 # guarantee that the SVN revision will exist for each of the dependant |
| 1907 # depots, so we have to grep the git logs and grab the next earlier one. | 2027 # depots, so we have to grep the git logs and grab the next earlier one. |
| 1908 if not is_base and\ | 2028 if (not is_base |
| 1909 DEPOT_DEPS_NAME[depot]['depends'] and\ | 2029 and DEPOT_DEPS_NAME[depot]['depends'] |
| 1910 self.source_control.IsGit(): | 2030 and self.source_control.IsGit()): |
| 1911 svn_rev = self.source_control.SVNFindRev(revision) | 2031 svn_rev = self.source_control.SVNFindRev(revision) |
| 1912 | 2032 |
| 1913 for d in DEPOT_DEPS_NAME[depot]['depends']: | 2033 for d in DEPOT_DEPS_NAME[depot]['depends']: |
| 1914 self.ChangeToDepotWorkingDirectory(d) | 2034 self.ChangeToDepotWorkingDirectory(d) |
| 1915 | 2035 |
| 1916 dependant_rev = self.source_control.ResolveToRevision( | 2036 dependant_rev = self.source_control.ResolveToRevision( |
| 1917 svn_rev, d, DEPOT_DEPS_NAME, -1000) | 2037 svn_rev, d, DEPOT_DEPS_NAME, -1000) |
| 1918 | 2038 |
| 1919 if dependant_rev: | 2039 if dependant_rev: |
| 1920 revisions_to_sync.append([d, dependant_rev]) | 2040 revisions_to_sync.append([d, dependant_rev]) |
| (...skipping 135 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2056 output = bisect_utils.CheckRunGit(cmd) | 2176 output = bisect_utils.CheckRunGit(cmd) |
| 2057 | 2177 |
| 2058 files = output.splitlines() | 2178 files = output.splitlines() |
| 2059 | 2179 |
| 2060 if len(files) == 1 and files[0] == 'DEPS': | 2180 if len(files) == 1 and files[0] == 'DEPS': |
| 2061 return True | 2181 return True |
| 2062 | 2182 |
| 2063 return False | 2183 return False |
| 2064 | 2184 |
| 2065 def SyncBuildAndRunRevision(self, revision, depot, command_to_run, metric, | 2185 def SyncBuildAndRunRevision(self, revision, depot, command_to_run, metric, |
| 2066 skippable=False): | 2186 skippable=False): |
| 2067 """Performs a full sync/build/run of the specified revision. | 2187 """Performs a full sync/build/run of the specified revision. |
| 2068 | 2188 |
| 2069 Args: | 2189 Args: |
| 2070 revision: The revision to sync to. | 2190 revision: The revision to sync to. |
| 2071 depot: The depot that's being used at the moment (src, webkit, etc.) | 2191 depot: The depot that's being used at the moment (src, webkit, etc.) |
| 2072 command_to_run: The command to execute the performance test. | 2192 command_to_run: The command to execute the performance test. |
| 2073 metric: The performance metric being tested. | 2193 metric: The performance metric being tested. |
| 2074 | 2194 |
| 2075 Returns: | 2195 Returns: |
| 2076 On success, a tuple containing the results of the performance test. | 2196 On success, a tuple containing the results of the performance test. |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2110 sync_client): | 2230 sync_client): |
| 2111 success = False | 2231 success = False |
| 2112 | 2232 |
| 2113 break | 2233 break |
| 2114 | 2234 |
| 2115 if success: | 2235 if success: |
| 2116 success = self.RunPostSync(depot) | 2236 success = self.RunPostSync(depot) |
| 2117 if success: | 2237 if success: |
| 2118 if skippable and self.ShouldSkipRevision(depot, revision): | 2238 if skippable and self.ShouldSkipRevision(depot, revision): |
| 2119 return ('Skipped revision: [%s]' % str(revision), | 2239 return ('Skipped revision: [%s]' % str(revision), |
| 2120 BUILD_RESULT_SKIPPED) | 2240 BUILD_RESULT_SKIPPED) |
| 2121 | 2241 |
| 2122 start_build_time = time.time() | 2242 start_build_time = time.time() |
| 2123 if self.BuildCurrentRevision(depot, revision): | 2243 if self.BuildCurrentRevision(depot, revision): |
| 2124 after_build_time = time.time() | 2244 after_build_time = time.time() |
| 2125 # Hack to support things that got changed. | 2245 # Hack to support things that got changed. |
| 2126 command_to_run = self.GetCompatibleCommand( | 2246 command_to_run = self.GetCompatibleCommand( |
| 2127 command_to_run, revision, depot) | 2247 command_to_run, revision, depot) |
| 2128 results = self.RunPerformanceTestAndParseResults(command_to_run, | 2248 results = self.RunPerformanceTestAndParseResults(command_to_run, |
| 2129 metric) | 2249 metric) |
| 2130 # Restore build output directory once the tests are done, to avoid | 2250 # Restore build output directory once the tests are done, to avoid |
| 2131 # any descrepancy. | 2251 # any descrepancy. |
| 2132 if self.IsDownloadable(depot) and revision: | 2252 if self.IsDownloadable(depot) and revision: |
| 2133 self.BackupOrRestoreOutputdirectory(restore=True) | 2253 self.BackupOrRestoreOutputdirectory(restore=True) |
| 2134 | 2254 |
| 2135 if results[1] == 0: | 2255 if results[1] == 0: |
| 2136 external_revisions = self.Get3rdPartyRevisionsFromCurrentRevision( | 2256 external_revisions = self._Get3rdPartyRevisions(depot) |
| 2137 depot, revision) | |
| 2138 | 2257 |
| 2139 if not external_revisions is None: | 2258 if not external_revisions is None: |
| 2140 return (results[0], results[1], external_revisions, | 2259 return (results[0], results[1], external_revisions, |
| 2141 time.time() - after_build_time, after_build_time - | 2260 time.time() - after_build_time, after_build_time - |
| 2142 start_build_time) | 2261 start_build_time) |
| 2143 else: | 2262 else: |
| 2144 return ('Failed to parse DEPS file for external revisions.', | 2263 return ('Failed to parse DEPS file for external revisions.', |
| 2145 BUILD_RESULT_FAIL) | 2264 BUILD_RESULT_FAIL) |
| 2146 else: | 2265 else: |
| 2147 return results | 2266 return results |
| 2148 else: | 2267 else: |
| 2149 return ('Failed to build revision: [%s]' % (str(revision, )), | 2268 return ('Failed to build revision: [%s]' % str(revision), |
| 2150 BUILD_RESULT_FAIL) | 2269 BUILD_RESULT_FAIL) |
| 2151 else: | 2270 else: |
| 2152 return ('Failed to run [gclient runhooks].', BUILD_RESULT_FAIL) | 2271 return ('Failed to run [gclient runhooks].', BUILD_RESULT_FAIL) |
| 2153 else: | 2272 else: |
| 2154 return ('Failed to sync revision: [%s]' % (str(revision, )), | 2273 return ('Failed to sync revision: [%s]' % str(revision), |
| 2155 BUILD_RESULT_FAIL) | 2274 BUILD_RESULT_FAIL) |
| 2156 | 2275 |
| 2157 def _CheckIfRunPassed(self, current_value, known_good_value, known_bad_value): | 2276 def _CheckIfRunPassed(self, current_value, known_good_value, known_bad_value): |
| 2158 """Given known good and bad values, decide if the current_value passed | 2277 """Given known good and bad values, decide if the current_value passed |
| 2159 or failed. | 2278 or failed. |
| 2160 | 2279 |
| 2161 Args: | 2280 Args: |
| 2162 current_value: The value of the metric being checked. | 2281 current_value: The value of the metric being checked. |
| 2163 known_bad_value: The reference value for a "failed" run. | 2282 known_bad_value: The reference value for a "failed" run. |
| 2164 known_good_value: The reference value for a "passed" run. | 2283 known_good_value: The reference value for a "passed" run. |
| 2165 | 2284 |
| (...skipping 13 matching lines...) Expand all Loading... |
| 2179 return dist_to_good_value < dist_to_bad_value | 2298 return dist_to_good_value < dist_to_bad_value |
| 2180 | 2299 |
| 2181 def _GetDepotDirectory(self, depot_name): | 2300 def _GetDepotDirectory(self, depot_name): |
| 2182 if depot_name == 'chromium': | 2301 if depot_name == 'chromium': |
| 2183 return self.src_cwd | 2302 return self.src_cwd |
| 2184 elif depot_name == 'cros': | 2303 elif depot_name == 'cros': |
| 2185 return self.cros_cwd | 2304 return self.cros_cwd |
| 2186 elif depot_name in DEPOT_NAMES: | 2305 elif depot_name in DEPOT_NAMES: |
| 2187 return self.depot_cwd[depot_name] | 2306 return self.depot_cwd[depot_name] |
| 2188 else: | 2307 else: |
| 2189 assert False, 'Unknown depot [ %s ] encountered. Possibly a new one'\ | 2308 assert False, ('Unknown depot [ %s ] encountered. Possibly a new one ' |
| 2190 ' was added without proper support?' % depot_name | 2309 'was added without proper support?' % depot_name) |
| 2191 | 2310 |
| 2192 def ChangeToDepotWorkingDirectory(self, depot_name): | 2311 def ChangeToDepotWorkingDirectory(self, depot_name): |
| 2193 """Given a depot, changes to the appropriate working directory. | 2312 """Given a depot, changes to the appropriate working directory. |
| 2194 | 2313 |
| 2195 Args: | 2314 Args: |
| 2196 depot_name: The name of the depot (see DEPOT_NAMES). | 2315 depot_name: The name of the depot (see DEPOT_NAMES). |
| 2197 """ | 2316 """ |
| 2198 os.chdir(self._GetDepotDirectory(depot_name)) | 2317 os.chdir(self._GetDepotDirectory(depot_name)) |
| 2199 | 2318 |
| 2200 def _FillInV8BleedingEdgeInfo(self, min_revision_data, max_revision_data): | 2319 def _FillInV8BleedingEdgeInfo(self, min_revision_data, max_revision_data): |
| 2201 r1 = self._GetNearestV8BleedingEdgeFromTrunk(min_revision_data['revision'], | 2320 r1 = self._GetNearestV8BleedingEdgeFromTrunk(min_revision_data['revision'], |
| 2202 search_forward=True) | 2321 search_forward=True) |
| 2203 r2 = self._GetNearestV8BleedingEdgeFromTrunk(max_revision_data['revision'], | 2322 r2 = self._GetNearestV8BleedingEdgeFromTrunk(max_revision_data['revision'], |
| 2204 search_forward=False) | 2323 search_forward=False) |
| 2205 min_revision_data['external']['v8_bleeding_edge'] = r1 | 2324 min_revision_data['external']['v8_bleeding_edge'] = r1 |
| 2206 max_revision_data['external']['v8_bleeding_edge'] = r2 | 2325 max_revision_data['external']['v8_bleeding_edge'] = r2 |
| 2207 | 2326 |
| 2208 if (not self._GetV8BleedingEdgeFromV8TrunkIfMappable( | 2327 if (not self._GetV8BleedingEdgeFromV8TrunkIfMappable( |
| 2209 min_revision_data['revision']) or | 2328 min_revision_data['revision']) |
| 2210 not self._GetV8BleedingEdgeFromV8TrunkIfMappable( | 2329 or not self._GetV8BleedingEdgeFromV8TrunkIfMappable( |
| 2211 max_revision_data['revision'])): | 2330 max_revision_data['revision'])): |
| 2212 self.warnings.append('Trunk revisions in V8 did not map directly to ' | 2331 self.warnings.append( |
| 2213 'bleeding_edge. Attempted to expand the range to find V8 rolls which ' | 2332 'Trunk revisions in V8 did not map directly to bleeding_edge. ' |
| 2214 'did map directly to bleeding_edge revisions, but results might not ' | 2333 'Attempted to expand the range to find V8 rolls which did map ' |
| 2215 'be valid.') | 2334 'directly to bleeding_edge revisions, but results might not be ' |
| 2335 'valid.') |
| 2216 | 2336 |
| 2217 def _FindNextDepotToBisect(self, current_depot, current_revision, | 2337 def _FindNextDepotToBisect( |
| 2218 min_revision_data, max_revision_data): | 2338 self, current_depot, min_revision_data, max_revision_data): |
| 2219 """Given the state of the bisect, decides which depot the script should | 2339 """Decides which depot the script should dive into next (if any). |
| 2220 dive into next (if any). | |
| 2221 | 2340 |
| 2222 Args: | 2341 Args: |
| 2223 current_depot: Current depot being bisected. | 2342 current_depot: Current depot being bisected. |
| 2224 current_revision: Current revision synced to. | |
| 2225 min_revision_data: Data about the earliest revision in the bisect range. | 2343 min_revision_data: Data about the earliest revision in the bisect range. |
| 2226 max_revision_data: Data about the latest revision in the bisect range. | 2344 max_revision_data: Data about the latest revision in the bisect range. |
| 2227 | 2345 |
| 2228 Returns: | 2346 Returns: |
| 2229 The depot to bisect next, or None. | 2347 Name of the depot to bisect next, or None. |
| 2230 """ | 2348 """ |
| 2231 external_depot = None | 2349 external_depot = None |
| 2232 for next_depot in DEPOT_NAMES: | 2350 for next_depot in DEPOT_NAMES: |
| 2233 if DEPOT_DEPS_NAME[next_depot].has_key('platform'): | 2351 if DEPOT_DEPS_NAME[next_depot].has_key('platform'): |
| 2234 if DEPOT_DEPS_NAME[next_depot]['platform'] != os.name: | 2352 if DEPOT_DEPS_NAME[next_depot]['platform'] != os.name: |
| 2235 continue | 2353 continue |
| 2236 | 2354 |
| 2237 if not (DEPOT_DEPS_NAME[next_depot]["recurse"] and | 2355 if not (DEPOT_DEPS_NAME[next_depot]['recurse'] |
| 2238 min_revision_data['depot'] in DEPOT_DEPS_NAME[next_depot]['from']): | 2356 and min_revision_data['depot'] |
| 2357 in DEPOT_DEPS_NAME[next_depot]['from']): |
| 2239 continue | 2358 continue |
| 2240 | 2359 |
| 2241 if current_depot == 'v8': | 2360 if current_depot == 'v8': |
| 2242 # We grab the bleeding_edge info here rather than earlier because we | 2361 # We grab the bleeding_edge info here rather than earlier because we |
| 2243 # finally have the revision range. From that we can search forwards and | 2362 # finally have the revision range. From that we can search forwards and |
| 2244 # backwards to try to match trunk revisions to bleeding_edge. | 2363 # backwards to try to match trunk revisions to bleeding_edge. |
| 2245 self._FillInV8BleedingEdgeInfo(min_revision_data, max_revision_data) | 2364 self._FillInV8BleedingEdgeInfo(min_revision_data, max_revision_data) |
| 2246 | 2365 |
| 2247 if (min_revision_data['external'].get(next_depot) == | 2366 if (min_revision_data['external'].get(next_depot) == |
| 2248 max_revision_data['external'].get(next_depot)): | 2367 max_revision_data['external'].get(next_depot)): |
| 2249 continue | 2368 continue |
| 2250 | 2369 |
| 2251 if (min_revision_data['external'].get(next_depot) and | 2370 if (min_revision_data['external'].get(next_depot) and |
| 2252 max_revision_data['external'].get(next_depot)): | 2371 max_revision_data['external'].get(next_depot)): |
| 2253 external_depot = next_depot | 2372 external_depot = next_depot |
| 2254 break | 2373 break |
| 2255 | 2374 |
| 2256 return external_depot | 2375 return external_depot |
| 2257 | 2376 |
| 2258 def PrepareToBisectOnDepot(self, | 2377 def PrepareToBisectOnDepot( |
| 2259 current_depot, | 2378 self, current_depot, end_revision, start_revision, previous_revision): |
| 2260 end_revision, | |
| 2261 start_revision, | |
| 2262 previous_depot, | |
| 2263 previous_revision): | |
| 2264 """Changes to the appropriate directory and gathers a list of revisions | 2379 """Changes to the appropriate directory and gathers a list of revisions |
| 2265 to bisect between |start_revision| and |end_revision|. | 2380 to bisect between |start_revision| and |end_revision|. |
| 2266 | 2381 |
| 2267 Args: | 2382 Args: |
| 2268 current_depot: The depot we want to bisect. | 2383 current_depot: The depot we want to bisect. |
| 2269 end_revision: End of the revision range. | 2384 end_revision: End of the revision range. |
| 2270 start_revision: Start of the revision range. | 2385 start_revision: Start of the revision range. |
| 2271 previous_depot: The depot we were previously bisecting. | |
| 2272 previous_revision: The last revision we synced to on |previous_depot|. | 2386 previous_revision: The last revision we synced to on |previous_depot|. |
| 2273 | 2387 |
| 2274 Returns: | 2388 Returns: |
| 2275 A list containing the revisions between |start_revision| and | 2389 A list containing the revisions between |start_revision| and |
| 2276 |end_revision| inclusive. | 2390 |end_revision| inclusive. |
| 2277 """ | 2391 """ |
| 2278 # Change into working directory of external library to run | 2392 # Change into working directory of external library to run |
| 2279 # subsequent commands. | 2393 # subsequent commands. |
| 2280 self.ChangeToDepotWorkingDirectory(current_depot) | 2394 self.ChangeToDepotWorkingDirectory(current_depot) |
| 2281 | 2395 |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2319 Args: | 2433 Args: |
| 2320 good_rev: The last known good revision where the performance regression | 2434 good_rev: The last known good revision where the performance regression |
| 2321 has not occurred yet. | 2435 has not occurred yet. |
| 2322 bad_rev: A revision where the performance regression has already occurred. | 2436 bad_rev: A revision where the performance regression has already occurred. |
| 2323 cmd: The command to execute the performance test. | 2437 cmd: The command to execute the performance test. |
| 2324 metric: The metric being tested for regression. | 2438 metric: The metric being tested for regression. |
| 2325 | 2439 |
| 2326 Returns: | 2440 Returns: |
| 2327 A tuple with the results of building and running each revision. | 2441 A tuple with the results of building and running each revision. |
| 2328 """ | 2442 """ |
| 2329 bad_run_results = self.SyncBuildAndRunRevision(bad_rev, | 2443 bad_run_results = self.SyncBuildAndRunRevision( |
| 2330 target_depot, | 2444 bad_rev, target_depot, cmd, metric) |
| 2331 cmd, | |
| 2332 metric) | |
| 2333 | 2445 |
| 2334 good_run_results = None | 2446 good_run_results = None |
| 2335 | 2447 |
| 2336 if not bad_run_results[1]: | 2448 if not bad_run_results[1]: |
| 2337 good_run_results = self.SyncBuildAndRunRevision(good_rev, | 2449 good_run_results = self.SyncBuildAndRunRevision( |
| 2338 target_depot, | 2450 good_rev, target_depot, cmd, metric) |
| 2339 cmd, | |
| 2340 metric) | |
| 2341 | 2451 |
| 2342 return (bad_run_results, good_run_results) | 2452 return (bad_run_results, good_run_results) |
| 2343 | 2453 |
| 2344 def AddRevisionsIntoRevisionData(self, revisions, depot, sort, revision_data): | |
| 2345 """Adds new revisions to the revision_data dict and initializes them. | |
| 2346 | |
| 2347 Args: | |
| 2348 revisions: List of revisions to add. | |
| 2349 depot: Depot that's currently in use (src, webkit, etc...) | |
| 2350 sort: Sorting key for displaying revisions. | |
| 2351 revision_data: A dict to add the new revisions into. Existing revisions | |
| 2352 will have their sort keys offset. | |
| 2353 """ | |
| 2354 | |
| 2355 num_depot_revisions = len(revisions) | |
| 2356 | |
| 2357 for _, v in revision_data.iteritems(): | |
| 2358 if v['sort'] > sort: | |
| 2359 v['sort'] += num_depot_revisions | |
| 2360 | |
| 2361 for i in xrange(num_depot_revisions): | |
| 2362 r = revisions[i] | |
| 2363 | |
| 2364 revision_data[r] = {'revision' : r, | |
| 2365 'depot' : depot, | |
| 2366 'value' : None, | |
| 2367 'perf_time' : 0, | |
| 2368 'build_time' : 0, | |
| 2369 'passed' : '?', | |
| 2370 'sort' : i + sort + 1} | |
| 2371 | |
| 2372 def PrintRevisionsToBisectMessage(self, revision_list, depot): | 2454 def PrintRevisionsToBisectMessage(self, revision_list, depot): |
| 2373 if self.opts.output_buildbot_annotations: | 2455 if self.opts.output_buildbot_annotations: |
| 2374 step_name = 'Bisection Range: [%s - %s]' % ( | 2456 step_name = 'Bisection Range: [%s - %s]' % ( |
| 2375 revision_list[len(revision_list)-1], revision_list[0]) | 2457 revision_list[len(revision_list)-1], revision_list[0]) |
| 2376 bisect_utils.OutputAnnotationStepStart(step_name) | 2458 bisect_utils.OutputAnnotationStepStart(step_name) |
| 2377 | 2459 |
| 2378 print | 2460 print |
| 2379 print 'Revisions to bisect on [%s]:' % depot | 2461 print 'Revisions to bisect on [%s]:' % depot |
| 2380 for revision_id in revision_list: | 2462 for revision_id in revision_list: |
| 2381 print ' -> %s' % (revision_id, ) | 2463 print ' -> %s' % (revision_id, ) |
| 2382 print | 2464 print |
| 2383 | 2465 |
| 2384 if self.opts.output_buildbot_annotations: | 2466 if self.opts.output_buildbot_annotations: |
| 2385 bisect_utils.OutputAnnotationStepClosed() | 2467 bisect_utils.OutputAnnotationStepClosed() |
| 2386 | 2468 |
| 2387 def NudgeRevisionsIfDEPSChange(self, bad_revision, good_revision): | 2469 def NudgeRevisionsIfDEPSChange(self, bad_revision, good_revision): |
| 2388 """Checks to see if changes to DEPS file occurred, and that the revision | 2470 """Checks to see if changes to DEPS file occurred, and that the revision |
| 2389 range also includes the change to .DEPS.git. If it doesn't, attempts to | 2471 range also includes the change to .DEPS.git. If it doesn't, attempts to |
| 2390 expand the revision range to include it. | 2472 expand the revision range to include it. |
| 2391 | 2473 |
| 2392 Args: | 2474 Args: |
| 2393 bad_rev: First known bad revision. | 2475 bad_rev: First known bad revision. |
| 2394 good_revision: Last known good revision. | 2476 good_revision: Last known good revision. |
| 2395 | 2477 |
| 2396 Returns: | 2478 Returns: |
| 2397 A tuple with the new bad and good revisions. | 2479 A tuple with the new bad and good revisions. |
| 2398 """ | 2480 """ |
| 2399 if self.source_control.IsGit() and self.opts.target_platform == 'chromium': | 2481 if self.source_control.IsGit() and self.opts.target_platform == 'chromium': |
| 2400 changes_to_deps = self.source_control.QueryFileRevisionHistory( | 2482 changes_to_deps = self.source_control.QueryFileRevisionHistory( |
| 2401 'DEPS', good_revision, bad_revision) | 2483 'DEPS', good_revision, bad_revision) |
| 2402 | 2484 |
| 2403 if changes_to_deps: | 2485 if changes_to_deps: |
| 2404 # DEPS file was changed, search from the oldest change to DEPS file to | 2486 # DEPS file was changed, search from the oldest change to DEPS file to |
| 2405 # bad_revision to see if there are matching .DEPS.git changes. | 2487 # bad_revision to see if there are matching .DEPS.git changes. |
| 2406 oldest_deps_change = changes_to_deps[-1] | 2488 oldest_deps_change = changes_to_deps[-1] |
| 2407 changes_to_gitdeps = self.source_control.QueryFileRevisionHistory( | 2489 changes_to_gitdeps = self.source_control.QueryFileRevisionHistory( |
| (...skipping 14 matching lines...) Expand all Loading... |
| 2422 output = output.strip() | 2504 output = output.strip() |
| 2423 if output: | 2505 if output: |
| 2424 self.warnings.append('Detected change to DEPS and modified ' | 2506 self.warnings.append('Detected change to DEPS and modified ' |
| 2425 'revision range to include change to .DEPS.git') | 2507 'revision range to include change to .DEPS.git') |
| 2426 return (output, good_revision) | 2508 return (output, good_revision) |
| 2427 else: | 2509 else: |
| 2428 self.warnings.append('Detected change to DEPS but couldn\'t find ' | 2510 self.warnings.append('Detected change to DEPS but couldn\'t find ' |
| 2429 'matching change to .DEPS.git') | 2511 'matching change to .DEPS.git') |
| 2430 return (bad_revision, good_revision) | 2512 return (bad_revision, good_revision) |
| 2431 | 2513 |
| 2432 def CheckIfRevisionsInProperOrder(self, | 2514 def CheckIfRevisionsInProperOrder( |
| 2433 target_depot, | 2515 self, target_depot, good_revision, bad_revision): |
| 2434 good_revision, | |
| 2435 bad_revision): | |
| 2436 """Checks that |good_revision| is an earlier revision than |bad_revision|. | 2516 """Checks that |good_revision| is an earlier revision than |bad_revision|. |
| 2437 | 2517 |
| 2438 Args: | 2518 Args: |
| 2439 good_revision: Number/tag of the known good revision. | 2519 good_revision: Number/tag of the known good revision. |
| 2440 bad_revision: Number/tag of the known bad revision. | 2520 bad_revision: Number/tag of the known bad revision. |
| 2441 | 2521 |
| 2442 Returns: | 2522 Returns: |
| 2443 True if the revisions are in the proper order (good earlier than bad). | 2523 True if the revisions are in the proper order (good earlier than bad). |
| 2444 """ | 2524 """ |
| 2445 if self.source_control.IsGit() and target_depot != 'cros': | 2525 if self.source_control.IsGit() and target_depot != 'cros': |
| 2446 cmd = ['log', '--format=%ct', '-1', good_revision] | 2526 cmd = ['log', '--format=%ct', '-1', good_revision] |
| 2447 cwd = self._GetDepotDirectory(target_depot) | 2527 cwd = self._GetDepotDirectory(target_depot) |
| 2448 | 2528 |
| 2449 output = bisect_utils.CheckRunGit(cmd, cwd=cwd) | 2529 output = bisect_utils.CheckRunGit(cmd, cwd=cwd) |
| 2450 good_commit_time = int(output) | 2530 good_commit_time = int(output) |
| 2451 | 2531 |
| 2452 cmd = ['log', '--format=%ct', '-1', bad_revision] | 2532 cmd = ['log', '--format=%ct', '-1', bad_revision] |
| 2453 output = bisect_utils.CheckRunGit(cmd, cwd=cwd) | 2533 output = bisect_utils.CheckRunGit(cmd, cwd=cwd) |
| (...skipping 29 matching lines...) Expand all Loading... |
| 2483 'built local ChromeShell(refer to crbug.com/385324).\n' | 2563 'built local ChromeShell(refer to crbug.com/385324).\n' |
| 2484 'Please try bisecting revisions greater than or equal to r265549.')} | 2564 'Please try bisecting revisions greater than or equal to r265549.')} |
| 2485 return None | 2565 return None |
| 2486 | 2566 |
| 2487 def Run(self, command_to_run, bad_revision_in, good_revision_in, metric): | 2567 def Run(self, command_to_run, bad_revision_in, good_revision_in, metric): |
| 2488 """Given known good and bad revisions, run a binary search on all | 2568 """Given known good and bad revisions, run a binary search on all |
| 2489 intermediate revisions to determine the CL where the performance regression | 2569 intermediate revisions to determine the CL where the performance regression |
| 2490 occurred. | 2570 occurred. |
| 2491 | 2571 |
| 2492 Args: | 2572 Args: |
| 2493 command_to_run: Specify the command to execute the performance test. | 2573 command_to_run: Specify the command to execute the performance test. |
| 2494 good_revision: Number/tag of the known good revision. | 2574 good_revision: Number/tag of the known good revision. |
| 2495 bad_revision: Number/tag of the known bad revision. | 2575 bad_revision: Number/tag of the known bad revision. |
| 2496 metric: The performance metric to monitor. | 2576 metric: The performance metric to monitor. |
| 2497 | 2577 |
| 2498 Returns: | 2578 Returns: |
| 2499 A dict with 2 members, 'revision_data' and 'error'. On success, | 2579 A dict with 2 members, 'revision_data' and 'error'. On success, |
| 2500 'revision_data' will contain a dict mapping revision ids to | 2580 'revision_data' will contain a dict mapping revision ids to |
| 2501 data about that revision. Each piece of revision data consists of a | 2581 data about that revision. Each piece of revision data consists of a |
| 2502 dict with the following keys: | 2582 dict with the following keys: |
| 2503 | 2583 |
| 2504 'passed': Represents whether the performance test was successful at | 2584 'passed': Represents whether the performance test was successful at |
| 2505 that revision. Possible values include: 1 (passed), 0 (failed), | 2585 that revision. Possible values include: 1 (passed), 0 (failed), |
| 2506 '?' (skipped), 'F' (build failed). | 2586 '?' (skipped), 'F' (build failed). |
| 2507 'depot': The depot that this revision is from (ie. WebKit) | 2587 'depot': The depot that this revision is from (ie. WebKit) |
| 2508 'external': If the revision is a 'src' revision, 'external' contains | 2588 'external': If the revision is a 'src' revision, 'external' contains |
| 2509 the revisions of each of the external libraries. | 2589 the revisions of each of the external libraries. |
| 2510 'sort': A sort value for sorting the dict in order of commits. | 2590 'sort': A sort value for sorting the dict in order of commits. |
| 2511 | 2591 |
| 2512 For example: | 2592 For example: |
| 2593 { |
| 2594 'error':None, |
| 2595 'revision_data': |
| 2513 { | 2596 { |
| 2514 'error':None, | 2597 'CL #1': |
| 2515 'revision_data': | |
| 2516 { | 2598 { |
| 2517 'CL #1': | 2599 'passed':False, |
| 2518 { | 2600 'depot':'chromium', |
| 2519 'passed':False, | 2601 'external':None, |
| 2520 'depot':'chromium', | 2602 'sort':0 |
| 2521 'external':None, | |
| 2522 'sort':0 | |
| 2523 } | |
| 2524 } | 2603 } |
| 2525 } | 2604 } |
| 2605 } |
| 2526 | 2606 |
| 2527 If an error occurred, the 'error' field will contain the message and | 2607 If an error occurred, the 'error' field will contain the message and |
| 2528 'revision_data' will be empty. | 2608 'revision_data' will be empty. |
| 2529 """ | 2609 """ |
| 2530 results = {'revision_data' : {}, | 2610 results = { |
| 2531 'error' : None} | 2611 'revision_data' : {}, |
| 2612 'error' : None, |
| 2613 } |
| 2532 | 2614 |
| 2533 # Choose depot to bisect first | 2615 # Choose depot to bisect first |
| 2534 target_depot = 'chromium' | 2616 target_depot = 'chromium' |
| 2535 if self.opts.target_platform == 'cros': | 2617 if self.opts.target_platform == 'cros': |
| 2536 target_depot = 'cros' | 2618 target_depot = 'cros' |
| 2537 elif self.opts.target_platform == 'android-chrome': | 2619 elif self.opts.target_platform == 'android-chrome': |
| 2538 target_depot = 'android-chrome' | 2620 target_depot = 'android-chrome' |
| 2539 | 2621 |
| 2540 cwd = os.getcwd() | 2622 cwd = os.getcwd() |
| 2541 self.ChangeToDepotWorkingDirectory(target_depot) | 2623 self.ChangeToDepotWorkingDirectory(target_depot) |
| 2542 | 2624 |
| 2543 # If they passed SVN CL's, etc... we can try match them to git SHA1's. | 2625 # If they passed SVN CL's, etc... we can try match them to git SHA1's. |
| 2544 bad_revision = self.source_control.ResolveToRevision( | 2626 bad_revision = self.source_control.ResolveToRevision( |
| 2545 bad_revision_in, target_depot, DEPOT_DEPS_NAME, 100) | 2627 bad_revision_in, target_depot, DEPOT_DEPS_NAME, 100) |
| 2546 good_revision = self.source_control.ResolveToRevision( | 2628 good_revision = self.source_control.ResolveToRevision( |
| 2547 good_revision_in, target_depot, DEPOT_DEPS_NAME, -100) | 2629 good_revision_in, target_depot, DEPOT_DEPS_NAME, -100) |
| 2548 | 2630 |
| 2549 os.chdir(cwd) | 2631 os.chdir(cwd) |
| 2550 | 2632 |
| 2551 | |
| 2552 if bad_revision is None: | 2633 if bad_revision is None: |
| 2553 results['error'] = 'Could\'t resolve [%s] to SHA1.' % (bad_revision_in,) | 2634 results['error'] = 'Could\'t resolve [%s] to SHA1.' % (bad_revision_in,) |
| 2554 return results | 2635 return results |
| 2555 | 2636 |
| 2556 if good_revision is None: | 2637 if good_revision is None: |
| 2557 results['error'] = 'Could\'t resolve [%s] to SHA1.' % (good_revision_in,) | 2638 results['error'] = 'Could\'t resolve [%s] to SHA1.' % (good_revision_in,) |
| 2558 return results | 2639 return results |
| 2559 | 2640 |
| 2560 # Check that they didn't accidentally swap good and bad revisions. | 2641 # Check that they didn't accidentally swap good and bad revisions. |
| 2561 if not self.CheckIfRevisionsInProperOrder( | 2642 if not self.CheckIfRevisionsInProperOrder( |
| 2562 target_depot, good_revision, bad_revision): | 2643 target_depot, good_revision, bad_revision): |
| 2563 results['error'] = 'bad_revision < good_revision, did you swap these '\ | 2644 results['error'] = ('bad_revision < good_revision, did you swap these ' |
| 2564 'by mistake?' | 2645 'by mistake?') |
| 2565 return results | 2646 return results |
| 2566 | 2647 |
| 2567 (bad_revision, good_revision) = self.NudgeRevisionsIfDEPSChange( | 2648 bad_revision, good_revision = self.NudgeRevisionsIfDEPSChange( |
| 2568 bad_revision, good_revision) | 2649 bad_revision, good_revision) |
| 2569 | 2650 |
| 2570 if self.opts.output_buildbot_annotations: | 2651 if self.opts.output_buildbot_annotations: |
| 2571 bisect_utils.OutputAnnotationStepStart('Gathering Revisions') | 2652 bisect_utils.OutputAnnotationStepStart('Gathering Revisions') |
| 2572 | 2653 |
| 2573 cannot_bisect = self.CanPerformBisect(good_revision) | 2654 cannot_bisect = self.CanPerformBisect(good_revision) |
| 2574 if cannot_bisect: | 2655 if cannot_bisect: |
| 2575 results['error'] = cannot_bisect.get('error') | 2656 results['error'] = cannot_bisect.get('error') |
| 2576 return results | 2657 return results |
| 2577 | 2658 |
| 2578 print 'Gathering revision range for bisection.' | 2659 print 'Gathering revision range for bisection.' |
| 2579 # Retrieve a list of revisions to do bisection on. | 2660 # Retrieve a list of revisions to do bisection on. |
| 2580 src_revision_list = self.GetRevisionList(target_depot, | 2661 src_revision_list = self.GetRevisionList( |
| 2581 bad_revision, | 2662 target_depot, bad_revision, good_revision) |
| 2582 good_revision) | |
| 2583 | 2663 |
| 2584 if self.opts.output_buildbot_annotations: | 2664 if self.opts.output_buildbot_annotations: |
| 2585 bisect_utils.OutputAnnotationStepClosed() | 2665 bisect_utils.OutputAnnotationStepClosed() |
| 2586 | 2666 |
| 2587 if src_revision_list: | 2667 if src_revision_list: |
| 2588 # revision_data will store information about a revision such as the | 2668 # revision_data will store information about a revision such as the |
| 2589 # depot it came from, the webkit/V8 revision at that time, | 2669 # depot it came from, the webkit/V8 revision at that time, |
| 2590 # performance timing, build state, etc... | 2670 # performance timing, build state, etc... |
| 2591 revision_data = results['revision_data'] | 2671 revision_data = results['revision_data'] |
| 2592 | 2672 |
| 2593 # revision_list is the list we're binary searching through at the moment. | 2673 # revision_list is the list we're binary searching through at the moment. |
| 2594 revision_list = [] | 2674 revision_list = [] |
| 2595 | 2675 |
| 2596 sort_key_ids = 0 | 2676 sort_key_ids = 0 |
| 2597 | 2677 |
| 2598 for current_revision_id in src_revision_list: | 2678 for current_revision_id in src_revision_list: |
| 2599 sort_key_ids += 1 | 2679 sort_key_ids += 1 |
| 2600 | 2680 |
| 2601 revision_data[current_revision_id] = {'value' : None, | 2681 revision_data[current_revision_id] = { |
| 2602 'passed' : '?', | 2682 'value' : None, |
| 2603 'depot' : target_depot, | 2683 'passed' : '?', |
| 2604 'external' : None, | 2684 'depot' : target_depot, |
| 2605 'perf_time' : 0, | 2685 'external' : None, |
| 2606 'build_time' : 0, | 2686 'perf_time' : 0, |
| 2607 'sort' : sort_key_ids} | 2687 'build_time' : 0, |
| 2688 'sort' : sort_key_ids, |
| 2689 } |
| 2608 revision_list.append(current_revision_id) | 2690 revision_list.append(current_revision_id) |
| 2609 | 2691 |
| 2610 min_revision = 0 | 2692 min_revision = 0 |
| 2611 max_revision = len(revision_list) - 1 | 2693 max_revision = len(revision_list) - 1 |
| 2612 | 2694 |
| 2613 self.PrintRevisionsToBisectMessage(revision_list, target_depot) | 2695 self.PrintRevisionsToBisectMessage(revision_list, target_depot) |
| 2614 | 2696 |
| 2615 if self.opts.output_buildbot_annotations: | 2697 if self.opts.output_buildbot_annotations: |
| 2616 bisect_utils.OutputAnnotationStepStart('Gathering Reference Values') | 2698 bisect_utils.OutputAnnotationStepStart('Gathering Reference Values') |
| 2617 | 2699 |
| 2618 print 'Gathering reference values for bisection.' | 2700 print 'Gathering reference values for bisection.' |
| 2619 | 2701 |
| 2620 # Perform the performance tests on the good and bad revisions, to get | 2702 # Perform the performance tests on the good and bad revisions, to get |
| 2621 # reference values. | 2703 # reference values. |
| 2622 (bad_results, good_results) = self.GatherReferenceValues(good_revision, | 2704 bad_results, good_results = self.GatherReferenceValues(good_revision, |
| 2623 bad_revision, | 2705 bad_revision, |
| 2624 command_to_run, | 2706 command_to_run, |
| 2625 metric, | 2707 metric, |
| 2626 target_depot) | 2708 target_depot) |
| 2627 | 2709 |
| 2628 if self.opts.output_buildbot_annotations: | 2710 if self.opts.output_buildbot_annotations: |
| 2629 bisect_utils.OutputAnnotationStepClosed() | 2711 bisect_utils.OutputAnnotationStepClosed() |
| 2630 | 2712 |
| 2631 if bad_results[1]: | 2713 if bad_results[1]: |
| 2632 results['error'] = ('An error occurred while building and running ' | 2714 results['error'] = ('An error occurred while building and running ' |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2676 if max_revision - min_revision <= 1: | 2758 if max_revision - min_revision <= 1: |
| 2677 current_depot = min_revision_data['depot'] | 2759 current_depot = min_revision_data['depot'] |
| 2678 if min_revision_data['passed'] == '?': | 2760 if min_revision_data['passed'] == '?': |
| 2679 next_revision_index = min_revision | 2761 next_revision_index = min_revision |
| 2680 elif max_revision_data['passed'] == '?': | 2762 elif max_revision_data['passed'] == '?': |
| 2681 next_revision_index = max_revision | 2763 next_revision_index = max_revision |
| 2682 elif current_depot in ['android-chrome', 'cros', 'chromium', 'v8']: | 2764 elif current_depot in ['android-chrome', 'cros', 'chromium', 'v8']: |
| 2683 previous_revision = revision_list[min_revision] | 2765 previous_revision = revision_list[min_revision] |
| 2684 # If there were changes to any of the external libraries we track, | 2766 # If there were changes to any of the external libraries we track, |
| 2685 # should bisect the changes there as well. | 2767 # should bisect the changes there as well. |
| 2686 external_depot = self._FindNextDepotToBisect(current_depot, | 2768 external_depot = self._FindNextDepotToBisect( |
| 2687 previous_revision, min_revision_data, max_revision_data) | 2769 current_depot, min_revision_data, max_revision_data) |
| 2688 | 2770 |
| 2689 # If there was no change in any of the external depots, the search | 2771 # If there was no change in any of the external depots, the search |
| 2690 # is over. | 2772 # is over. |
| 2691 if not external_depot: | 2773 if not external_depot: |
| 2692 if current_depot == 'v8': | 2774 if current_depot == 'v8': |
| 2693 self.warnings.append('Unfortunately, V8 bisection couldn\'t ' | 2775 self.warnings.append('Unfortunately, V8 bisection couldn\'t ' |
| 2694 'continue any further. The script can only bisect into ' | 2776 'continue any further. The script can only bisect into ' |
| 2695 'V8\'s bleeding_edge repository if both the current and ' | 2777 'V8\'s bleeding_edge repository if both the current and ' |
| 2696 'previous revisions in trunk map directly to revisions in ' | 2778 'previous revisions in trunk map directly to revisions in ' |
| 2697 'bleeding_edge.') | 2779 'bleeding_edge.') |
| 2698 break | 2780 break |
| 2699 | 2781 |
| 2700 earliest_revision = max_revision_data['external'][external_depot] | 2782 earliest_revision = max_revision_data['external'][external_depot] |
| 2701 latest_revision = min_revision_data['external'][external_depot] | 2783 latest_revision = min_revision_data['external'][external_depot] |
| 2702 | 2784 |
| 2703 new_revision_list = self.PrepareToBisectOnDepot(external_depot, | 2785 new_revision_list = self.PrepareToBisectOnDepot( |
| 2704 latest_revision, | 2786 external_depot, latest_revision, earliest_revision, |
| 2705 earliest_revision, | 2787 previous_revision) |
| 2706 next_revision_depot, | |
| 2707 previous_revision) | |
| 2708 | 2788 |
| 2709 if not new_revision_list: | 2789 if not new_revision_list: |
| 2710 results['error'] = 'An error occurred attempting to retrieve'\ | 2790 results['error'] = ('An error occurred attempting to retrieve ' |
| 2711 ' revision range: [%s..%s]' % \ | 2791 'revision range: [%s..%s]' % |
| 2712 (earliest_revision, latest_revision) | 2792 (earliest_revision, latest_revision)) |
| 2713 return results | 2793 return results |
| 2714 | 2794 |
| 2715 self.AddRevisionsIntoRevisionData(new_revision_list, | 2795 _AddRevisionsIntoRevisionData( |
| 2716 external_depot, | 2796 new_revision_list, external_depot, min_revision_data['sort'], |
| 2717 min_revision_data['sort'], | 2797 revision_data) |
| 2718 revision_data) | |
| 2719 | 2798 |
| 2720 # Reset the bisection and perform it on the newly inserted | 2799 # Reset the bisection and perform it on the newly inserted |
| 2721 # changelists. | 2800 # changelists. |
| 2722 revision_list = new_revision_list | 2801 revision_list = new_revision_list |
| 2723 min_revision = 0 | 2802 min_revision = 0 |
| 2724 max_revision = len(revision_list) - 1 | 2803 max_revision = len(revision_list) - 1 |
| 2725 sort_key_ids += len(revision_list) | 2804 sort_key_ids += len(revision_list) |
| 2726 | 2805 |
| 2727 print 'Regression in metric:%s appears to be the result of changes'\ | 2806 print ('Regression in metric %s appears to be the result of ' |
| 2728 ' in [%s].' % (metric, external_depot) | 2807 'changes in [%s].' % (metric, external_depot)) |
| 2729 | 2808 |
| 2730 self.PrintRevisionsToBisectMessage(revision_list, external_depot) | 2809 self.PrintRevisionsToBisectMessage(revision_list, external_depot) |
| 2731 | 2810 |
| 2732 continue | 2811 continue |
| 2733 else: | 2812 else: |
| 2734 break | 2813 break |
| 2735 else: | 2814 else: |
| 2736 next_revision_index = int((max_revision - min_revision) / 2) +\ | 2815 next_revision_index = (int((max_revision - min_revision) / 2) + |
| 2737 min_revision | 2816 min_revision) |
| 2738 | 2817 |
| 2739 next_revision_id = revision_list[next_revision_index] | 2818 next_revision_id = revision_list[next_revision_index] |
| 2740 next_revision_data = revision_data[next_revision_id] | 2819 next_revision_data = revision_data[next_revision_id] |
| 2741 next_revision_depot = next_revision_data['depot'] | 2820 next_revision_depot = next_revision_data['depot'] |
| 2742 | 2821 |
| 2743 self.ChangeToDepotWorkingDirectory(next_revision_depot) | 2822 self.ChangeToDepotWorkingDirectory(next_revision_depot) |
| 2744 | 2823 |
| 2745 if self.opts.output_buildbot_annotations: | 2824 if self.opts.output_buildbot_annotations: |
| 2746 step_name = 'Working on [%s]' % next_revision_id | 2825 step_name = 'Working on [%s]' % next_revision_id |
| 2747 bisect_utils.OutputAnnotationStepStart(step_name) | 2826 bisect_utils.OutputAnnotationStepStart(step_name) |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2783 # If the build is broken, remove it and redo search. | 2862 # If the build is broken, remove it and redo search. |
| 2784 revision_list.pop(next_revision_index) | 2863 revision_list.pop(next_revision_index) |
| 2785 | 2864 |
| 2786 max_revision -= 1 | 2865 max_revision -= 1 |
| 2787 | 2866 |
| 2788 if self.opts.output_buildbot_annotations: | 2867 if self.opts.output_buildbot_annotations: |
| 2789 self._PrintPartialResults(results) | 2868 self._PrintPartialResults(results) |
| 2790 bisect_utils.OutputAnnotationStepClosed() | 2869 bisect_utils.OutputAnnotationStepClosed() |
| 2791 else: | 2870 else: |
| 2792 # Weren't able to sync and retrieve the revision range. | 2871 # Weren't able to sync and retrieve the revision range. |
| 2793 results['error'] = 'An error occurred attempting to retrieve revision '\ | 2872 results['error'] = ('An error occurred attempting to retrieve revision ' |
| 2794 'range: [%s..%s]' % (good_revision, bad_revision) | 2873 'range: [%s..%s]' % (good_revision, bad_revision)) |
| 2795 | 2874 |
| 2796 return results | 2875 return results |
| 2797 | 2876 |
| 2798 def _PrintPartialResults(self, results_dict): | 2877 def _PrintPartialResults(self, results_dict): |
| 2799 revision_data = results_dict['revision_data'] | 2878 revision_data = results_dict['revision_data'] |
| 2800 revision_data_sorted = sorted(revision_data.iteritems(), | 2879 revision_data_sorted = sorted(revision_data.iteritems(), |
| 2801 key = lambda x: x[1]['sort']) | 2880 key = lambda x: x[1]['sort']) |
| 2802 results_dict = self._GetResultsDict(revision_data, revision_data_sorted) | 2881 results_dict = self._GetResultsDict(revision_data, revision_data_sorted) |
| 2803 | 2882 |
| 2804 self._PrintTestedCommitsTable(revision_data_sorted, | 2883 self._PrintTestedCommitsTable(revision_data_sorted, |
| 2805 results_dict['first_working_revision'], | 2884 results_dict['first_working_revision'], |
| 2806 results_dict['last_broken_revision'], | 2885 results_dict['last_broken_revision'], |
| 2807 100, final_step=False) | 2886 100, final_step=False) |
| 2808 | 2887 |
| 2809 def _PrintConfidence(self, results_dict): | |
| 2810 # The perf dashboard specifically looks for the string | |
| 2811 # "Confidence in Bisection Results: 100%" to decide whether or not | |
| 2812 # to cc the author(s). If you change this, please update the perf | |
| 2813 # dashboard as well. | |
| 2814 print 'Confidence in Bisection Results: %d%%' % results_dict['confidence'] | |
| 2815 | |
| 2816 def _ConfidenceLevelStatus(self, results_dict): | 2888 def _ConfidenceLevelStatus(self, results_dict): |
| 2817 if not results_dict['confidence']: | 2889 if not results_dict['confidence']: |
| 2818 return None | 2890 return None |
| 2819 confidence_status = 'Successful with %(level)s confidence%(warning)s.' | 2891 confidence_status = 'Successful with %(level)s confidence%(warning)s.' |
| 2820 if results_dict['confidence'] >= 95: | 2892 if results_dict['confidence'] >= 95: |
| 2821 level = 'high' | 2893 level = 'high' |
| 2822 else: | 2894 else: |
| 2823 level = 'low' | 2895 level = 'low' |
| 2824 warning = ' and warnings' | 2896 warning = ' and warnings' |
| 2825 if not self.warnings: | 2897 if not self.warnings: |
| 2826 warning = '' | 2898 warning = '' |
| 2827 return confidence_status % {'level': level, 'warning': warning} | 2899 return confidence_status % {'level': level, 'warning': warning} |
| 2828 | 2900 |
| 2829 def _PrintThankYou(self): | |
| 2830 print RESULTS_THANKYOU | |
| 2831 | |
| 2832 def _PrintBanner(self, results_dict): | |
| 2833 if self._IsBisectModeReturnCode(): | |
| 2834 metrics = 'N/A' | |
| 2835 change = 'Yes' | |
| 2836 else: | |
| 2837 metrics = '/'.join(self.opts.metric) | |
| 2838 change = '%.02f%% (+/-%.02f%%)' % ( | |
| 2839 results_dict['regression_size'], results_dict['regression_std_err']) | |
| 2840 | |
| 2841 if results_dict['culprit_revisions'] and results_dict['confidence']: | |
| 2842 status = self._ConfidenceLevelStatus(results_dict) | |
| 2843 else: | |
| 2844 status = 'Failure, could not reproduce.' | |
| 2845 change = 'Bisect could not reproduce a change.' | |
| 2846 | |
| 2847 print RESULTS_BANNER % { | |
| 2848 'status': status, | |
| 2849 'command': self.opts.command, | |
| 2850 'metrics': metrics, | |
| 2851 'change': change, | |
| 2852 'confidence': results_dict['confidence'], | |
| 2853 } | |
| 2854 | |
| 2855 | |
| 2856 def _PrintFailedBanner(self, results_dict): | |
| 2857 print | |
| 2858 if self._IsBisectModeReturnCode(): | |
| 2859 print 'Bisect could not reproduce a change in the return code.' | |
| 2860 else: | |
| 2861 print ('Bisect could not reproduce a change in the ' | |
| 2862 '%s metric.' % '/'.join(self.opts.metric)) | |
| 2863 print | |
| 2864 | |
| 2865 def _GetViewVCLinkFromDepotAndHash(self, cl, depot): | 2901 def _GetViewVCLinkFromDepotAndHash(self, cl, depot): |
| 2866 info = self.source_control.QueryRevisionInfo(cl, | 2902 info = self.source_control.QueryRevisionInfo(cl, |
| 2867 self._GetDepotDirectory(depot)) | 2903 self._GetDepotDirectory(depot)) |
| 2868 if depot and DEPOT_DEPS_NAME[depot].has_key('viewvc'): | 2904 if depot and DEPOT_DEPS_NAME[depot].has_key('viewvc'): |
| 2869 try: | 2905 try: |
| 2870 # Format is "git-svn-id: svn://....@123456 <other data>" | 2906 # Format is "git-svn-id: svn://....@123456 <other data>" |
| 2871 svn_line = [i for i in info['body'].splitlines() if 'git-svn-id:' in i] | 2907 svn_line = [i for i in info['body'].splitlines() if 'git-svn-id:' in i] |
| 2872 svn_revision = svn_line[0].split('@') | 2908 svn_revision = svn_line[0].split('@') |
| 2873 svn_revision = svn_revision[1].split(' ')[0] | 2909 svn_revision = svn_revision[1].split(' ')[0] |
| 2874 return DEPOT_DEPS_NAME[depot]['viewvc'] + svn_revision | 2910 return DEPOT_DEPS_NAME[depot]['viewvc'] + svn_revision |
| (...skipping 11 matching lines...) Expand all Loading... |
| 2886 else: | 2922 else: |
| 2887 commit_info = ('\nFailed to parse svn revision from body:\n%s' % | 2923 commit_info = ('\nFailed to parse svn revision from body:\n%s' % |
| 2888 info['body']) | 2924 info['body']) |
| 2889 print RESULTS_REVISION_INFO % { | 2925 print RESULTS_REVISION_INFO % { |
| 2890 'subject': info['subject'], | 2926 'subject': info['subject'], |
| 2891 'author': info['author'], | 2927 'author': info['author'], |
| 2892 'email_info': email_info, | 2928 'email_info': email_info, |
| 2893 'commit_info': commit_info, | 2929 'commit_info': commit_info, |
| 2894 'cl': cl, | 2930 'cl': cl, |
| 2895 'cl_date': info['date'] | 2931 'cl_date': info['date'] |
| 2896 } | 2932 } |
| 2897 | |
| 2898 def _PrintTableRow(self, column_widths, row_data): | |
| 2899 assert len(column_widths) == len(row_data) | |
| 2900 | |
| 2901 text = '' | |
| 2902 for i in xrange(len(column_widths)): | |
| 2903 current_row_data = row_data[i].center(column_widths[i], ' ') | |
| 2904 text += ('%%%ds' % column_widths[i]) % current_row_data | |
| 2905 print text | |
| 2906 | 2933 |
| 2907 def _PrintTestedCommitsHeader(self): | 2934 def _PrintTestedCommitsHeader(self): |
| 2908 if self.opts.bisect_mode == BISECT_MODE_MEAN: | 2935 if self.opts.bisect_mode == BISECT_MODE_MEAN: |
| 2909 self._PrintTableRow( | 2936 _PrintTableRow( |
| 2910 [20, 70, 14, 12, 13], | 2937 [20, 70, 14, 12, 13], |
| 2911 ['Depot', 'Commit SHA', 'Mean', 'Std. Error', 'State']) | 2938 ['Depot', 'Commit SHA', 'Mean', 'Std. Error', 'State']) |
| 2912 elif self.opts.bisect_mode == BISECT_MODE_STD_DEV: | 2939 elif self.opts.bisect_mode == BISECT_MODE_STD_DEV: |
| 2913 self._PrintTableRow( | 2940 _PrintTableRow( |
| 2914 [20, 70, 14, 12, 13], | 2941 [20, 70, 14, 12, 13], |
| 2915 ['Depot', 'Commit SHA', 'Std. Error', 'Mean', 'State']) | 2942 ['Depot', 'Commit SHA', 'Std. Error', 'Mean', 'State']) |
| 2916 elif self.opts.bisect_mode == BISECT_MODE_RETURN_CODE: | 2943 elif self.opts.bisect_mode == BISECT_MODE_RETURN_CODE: |
| 2917 self._PrintTableRow( | 2944 _PrintTableRow( |
| 2918 [20, 70, 14, 13], | 2945 [20, 70, 14, 13], |
| 2919 ['Depot', 'Commit SHA', 'Return Code', 'State']) | 2946 ['Depot', 'Commit SHA', 'Return Code', 'State']) |
| 2920 else: | 2947 else: |
| 2921 assert False, "Invalid bisect_mode specified." | 2948 assert False, 'Invalid bisect_mode specified.' |
| 2922 print ' %20s %70s %14s %13s' % ('Depot'.center(20, ' '), | |
| 2923 'Commit SHA'.center(70, ' '), 'Return Code'.center(14, ' '), | |
| 2924 'State'.center(13, ' ')) | |
| 2925 | 2949 |
| 2926 def _PrintTestedCommitsEntry(self, current_data, cl_link, state_str): | 2950 def _PrintTestedCommitsEntry(self, current_data, cl_link, state_str): |
| 2927 if self.opts.bisect_mode == BISECT_MODE_MEAN: | 2951 if self.opts.bisect_mode == BISECT_MODE_MEAN: |
| 2928 std_error = '+-%.02f' % current_data['value']['std_err'] | 2952 std_error = '+-%.02f' % current_data['value']['std_err'] |
| 2929 mean = '%.02f' % current_data['value']['mean'] | 2953 mean = '%.02f' % current_data['value']['mean'] |
| 2930 self._PrintTableRow( | 2954 _PrintTableRow( |
| 2931 [20, 70, 12, 14, 13], | 2955 [20, 70, 12, 14, 13], |
| 2932 [current_data['depot'], cl_link, mean, std_error, state_str]) | 2956 [current_data['depot'], cl_link, mean, std_error, state_str]) |
| 2933 elif self.opts.bisect_mode == BISECT_MODE_STD_DEV: | 2957 elif self.opts.bisect_mode == BISECT_MODE_STD_DEV: |
| 2934 std_error = '+-%.02f' % current_data['value']['std_err'] | 2958 std_error = '+-%.02f' % current_data['value']['std_err'] |
| 2935 mean = '%.02f' % current_data['value']['mean'] | 2959 mean = '%.02f' % current_data['value']['mean'] |
| 2936 self._PrintTableRow( | 2960 _PrintTableRow( |
| 2937 [20, 70, 12, 14, 13], | 2961 [20, 70, 12, 14, 13], |
| 2938 [current_data['depot'], cl_link, std_error, mean, state_str]) | 2962 [current_data['depot'], cl_link, std_error, mean, state_str]) |
| 2939 elif self.opts.bisect_mode == BISECT_MODE_RETURN_CODE: | 2963 elif self.opts.bisect_mode == BISECT_MODE_RETURN_CODE: |
| 2940 mean = '%d' % current_data['value']['mean'] | 2964 mean = '%d' % current_data['value']['mean'] |
| 2941 self._PrintTableRow( | 2965 _PrintTableRow( |
| 2942 [20, 70, 14, 13], | 2966 [20, 70, 14, 13], |
| 2943 [current_data['depot'], cl_link, mean, state_str]) | 2967 [current_data['depot'], cl_link, mean, state_str]) |
| 2944 | 2968 |
| 2945 def _PrintTestedCommitsTable(self, revision_data_sorted, | 2969 def _PrintTestedCommitsTable( |
| 2946 first_working_revision, last_broken_revision, confidence, | 2970 self, revision_data_sorted, first_working_revision, last_broken_revision, |
| 2947 final_step=True): | 2971 confidence, final_step=True): |
| 2948 print | 2972 print |
| 2949 if final_step: | 2973 if final_step: |
| 2950 print '===== TESTED COMMITS =====' | 2974 print '===== TESTED COMMITS =====' |
| 2951 else: | 2975 else: |
| 2952 print '===== PARTIAL RESULTS =====' | 2976 print '===== PARTIAL RESULTS =====' |
| 2953 self._PrintTestedCommitsHeader() | 2977 self._PrintTestedCommitsHeader() |
| 2954 state = 0 | 2978 state = 0 |
| 2955 for current_id, current_data in revision_data_sorted: | 2979 for current_id, current_data in revision_data_sorted: |
| 2956 if current_data['value']: | 2980 if current_data['value']: |
| 2957 if (current_id == last_broken_revision or | 2981 if (current_id == last_broken_revision or |
| (...skipping 18 matching lines...) Expand all Loading... |
| 2976 state_str = '' | 3000 state_str = '' |
| 2977 state_str = state_str.center(13, ' ') | 3001 state_str = state_str.center(13, ' ') |
| 2978 | 3002 |
| 2979 cl_link = self._GetViewVCLinkFromDepotAndHash(current_id, | 3003 cl_link = self._GetViewVCLinkFromDepotAndHash(current_id, |
| 2980 current_data['depot']) | 3004 current_data['depot']) |
| 2981 if not cl_link: | 3005 if not cl_link: |
| 2982 cl_link = current_id | 3006 cl_link = current_id |
| 2983 self._PrintTestedCommitsEntry(current_data, cl_link, state_str) | 3007 self._PrintTestedCommitsEntry(current_data, cl_link, state_str) |
| 2984 | 3008 |
| 2985 def _PrintReproSteps(self): | 3009 def _PrintReproSteps(self): |
| 3010 """Prints out a section of the results explaining how to run the test. |
| 3011 |
| 3012 This message includes the command used to run the test. |
| 3013 """ |
| 2986 command = '$ ' + self.opts.command | 3014 command = '$ ' + self.opts.command |
| 2987 if bisect_utils.IsTelemetryCommand(self.opts.command): | 3015 if bisect_utils.IsTelemetryCommand(self.opts.command): |
| 2988 command += ('\nAlso consider passing --profiler=list to see available ' | 3016 command += ('\nAlso consider passing --profiler=list to see available ' |
| 2989 'profilers.') | 3017 'profilers.') |
| 2990 print REPRO_STEPS_LOCAL % {'command': command} | 3018 print REPRO_STEPS_LOCAL % {'command': command} |
| 2991 print REPRO_STEPS_TRYJOB % {'command': command} | 3019 print REPRO_STEPS_TRYJOB % {'command': command} |
| 2992 | 3020 |
| 2993 def _PrintOtherRegressions(self, other_regressions, revision_data): | 3021 def _PrintOtherRegressions(self, other_regressions, revision_data): |
| 3022 """Prints a section of the results about other potential regressions.""" |
| 2994 print | 3023 print |
| 2995 print 'Other regressions may have occurred:' | 3024 print 'Other regressions may have occurred:' |
| 2996 print ' %8s %70s %10s' % ('Depot'.center(8, ' '), | 3025 print ' %8s %70s %10s' % ('Depot'.center(8, ' '), |
| 2997 'Range'.center(70, ' '), 'Confidence'.center(10, ' ')) | 3026 'Range'.center(70, ' '), 'Confidence'.center(10, ' ')) |
| 2998 for regression in other_regressions: | 3027 for regression in other_regressions: |
| 2999 current_id, previous_id, confidence = regression | 3028 current_id, previous_id, confidence = regression |
| 3000 current_data = revision_data[current_id] | 3029 current_data = revision_data[current_id] |
| 3001 previous_data = revision_data[previous_id] | 3030 previous_data = revision_data[previous_id] |
| 3002 | 3031 |
| 3003 current_link = self._GetViewVCLinkFromDepotAndHash(current_id, | 3032 current_link = self._GetViewVCLinkFromDepotAndHash(current_id, |
| 3004 current_data['depot']) | 3033 current_data['depot']) |
| 3005 previous_link = self._GetViewVCLinkFromDepotAndHash(previous_id, | 3034 previous_link = self._GetViewVCLinkFromDepotAndHash(previous_id, |
| 3006 previous_data['depot']) | 3035 previous_data['depot']) |
| 3007 | 3036 |
| 3008 # If we can't map it to a viewable URL, at least show the original hash. | 3037 # If we can't map it to a viewable URL, at least show the original hash. |
| 3009 if not current_link: | 3038 if not current_link: |
| 3010 current_link = current_id | 3039 current_link = current_id |
| 3011 if not previous_link: | 3040 if not previous_link: |
| 3012 previous_link = previous_id | 3041 previous_link = previous_id |
| 3013 | 3042 |
| 3014 print ' %8s %70s %s' % ( | 3043 print ' %8s %70s %s' % ( |
| 3015 current_data['depot'], current_link, | 3044 current_data['depot'], current_link, |
| 3016 ('%d%%' % confidence).center(10, ' ')) | 3045 ('%d%%' % confidence).center(10, ' ')) |
| 3017 print ' %8s %70s' % ( | 3046 print ' %8s %70s' % ( |
| 3018 previous_data['depot'], previous_link) | 3047 previous_data['depot'], previous_link) |
| 3019 print | 3048 print |
| 3020 | 3049 |
| 3021 def _PrintStepTime(self, revision_data_sorted): | |
| 3022 step_perf_time_avg = 0.0 | |
| 3023 step_build_time_avg = 0.0 | |
| 3024 step_count = 0.0 | |
| 3025 for _, current_data in revision_data_sorted: | |
| 3026 if current_data['value']: | |
| 3027 step_perf_time_avg += current_data['perf_time'] | |
| 3028 step_build_time_avg += current_data['build_time'] | |
| 3029 step_count += 1 | |
| 3030 if step_count: | |
| 3031 step_perf_time_avg = step_perf_time_avg / step_count | |
| 3032 step_build_time_avg = step_build_time_avg / step_count | |
| 3033 print | |
| 3034 print 'Average build time : %s' % datetime.timedelta( | |
| 3035 seconds=int(step_build_time_avg)) | |
| 3036 print 'Average test time : %s' % datetime.timedelta( | |
| 3037 seconds=int(step_perf_time_avg)) | |
| 3038 | |
| 3039 def _PrintWarnings(self): | |
| 3040 if not self.warnings: | |
| 3041 return | |
| 3042 print | |
| 3043 print 'WARNINGS:' | |
| 3044 for w in set(self.warnings): | |
| 3045 print ' ! %s' % w | |
| 3046 | |
| 3047 def _FindOtherRegressions(self, revision_data_sorted, bad_greater_than_good): | |
| 3048 other_regressions = [] | |
| 3049 previous_values = [] | |
| 3050 previous_id = None | |
| 3051 for current_id, current_data in revision_data_sorted: | |
| 3052 current_values = current_data['value'] | |
| 3053 if current_values: | |
| 3054 current_values = current_values['values'] | |
| 3055 if previous_values: | |
| 3056 confidence = ConfidenceScore(previous_values, [current_values]) | |
| 3057 mean_of_prev_runs = math_utils.Mean(sum(previous_values, [])) | |
| 3058 mean_of_current_runs = math_utils.Mean(current_values) | |
| 3059 | |
| 3060 # Check that the potential regression is in the same direction as | |
| 3061 # the overall regression. If the mean of the previous runs < the | |
| 3062 # mean of the current runs, this local regression is in same | |
| 3063 # direction. | |
| 3064 prev_less_than_current = mean_of_prev_runs < mean_of_current_runs | |
| 3065 is_same_direction = (prev_less_than_current if | |
| 3066 bad_greater_than_good else not prev_less_than_current) | |
| 3067 | |
| 3068 # Only report potential regressions with high confidence. | |
| 3069 if is_same_direction and confidence > 50: | |
| 3070 other_regressions.append([current_id, previous_id, confidence]) | |
| 3071 previous_values.append(current_values) | |
| 3072 previous_id = current_id | |
| 3073 return other_regressions | |
| 3074 | |
| 3075 | |
| 3076 def _GetResultsDict(self, revision_data, revision_data_sorted): | 3050 def _GetResultsDict(self, revision_data, revision_data_sorted): |
| 3077 # Find range where it possibly broke. | 3051 # Find range where it possibly broke. |
| 3078 first_working_revision = None | 3052 first_working_revision = None |
| 3079 first_working_revision_index = -1 | 3053 first_working_revision_index = -1 |
| 3080 last_broken_revision = None | 3054 last_broken_revision = None |
| 3081 last_broken_revision_index = -1 | 3055 last_broken_revision_index = -1 |
| 3082 | 3056 |
| 3083 for i in xrange(len(revision_data_sorted)): | 3057 for i in xrange(len(revision_data_sorted)): |
| 3084 k, v = revision_data_sorted[i] | 3058 k, v = revision_data_sorted[i] |
| 3085 if v['passed'] == 1: | 3059 if v['passed'] == 1: |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3129 cwd = os.getcwd() | 3103 cwd = os.getcwd() |
| 3130 self.ChangeToDepotWorkingDirectory( | 3104 self.ChangeToDepotWorkingDirectory( |
| 3131 revision_data[last_broken_revision]['depot']) | 3105 revision_data[last_broken_revision]['depot']) |
| 3132 | 3106 |
| 3133 if revision_data[last_broken_revision]['depot'] == 'cros': | 3107 if revision_data[last_broken_revision]['depot'] == 'cros': |
| 3134 # Want to get a list of all the commits and what depots they belong | 3108 # Want to get a list of all the commits and what depots they belong |
| 3135 # to so that we can grab info about each. | 3109 # to so that we can grab info about each. |
| 3136 cmd = ['repo', 'forall', '-c', | 3110 cmd = ['repo', 'forall', '-c', |
| 3137 'pwd ; git log --pretty=oneline --before=%d --after=%d' % ( | 3111 'pwd ; git log --pretty=oneline --before=%d --after=%d' % ( |
| 3138 last_broken_revision, first_working_revision + 1)] | 3112 last_broken_revision, first_working_revision + 1)] |
| 3139 (output, return_code) = bisect_utils.RunProcessAndRetrieveOutput(cmd) | 3113 output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd) |
| 3140 | 3114 |
| 3141 changes = [] | 3115 changes = [] |
| 3142 assert not return_code, 'An error occurred while running'\ | 3116 assert not return_code, ('An error occurred while running ' |
| 3143 ' "%s"' % ' '.join(cmd) | 3117 '"%s"' % ' '.join(cmd)) |
| 3144 last_depot = None | 3118 last_depot = None |
| 3145 cwd = os.getcwd() | 3119 cwd = os.getcwd() |
| 3146 for l in output.split('\n'): | 3120 for l in output.split('\n'): |
| 3147 if l: | 3121 if l: |
| 3148 # Output will be in form: | 3122 # Output will be in form: |
| 3149 # /path_to_depot | 3123 # /path_to_depot |
| 3150 # /path_to_other_depot | 3124 # /path_to_other_depot |
| 3151 # <SHA1> | 3125 # <SHA1> |
| 3152 # /path_again | 3126 # /path_again |
| 3153 # <SHA1> | 3127 # <SHA1> |
| (...skipping 11 matching lines...) Expand all Loading... |
| 3165 else: | 3139 else: |
| 3166 for i in xrange(last_broken_revision_index, len(revision_data_sorted)): | 3140 for i in xrange(last_broken_revision_index, len(revision_data_sorted)): |
| 3167 k, v = revision_data_sorted[i] | 3141 k, v = revision_data_sorted[i] |
| 3168 if k == first_working_revision: | 3142 if k == first_working_revision: |
| 3169 break | 3143 break |
| 3170 self.ChangeToDepotWorkingDirectory(v['depot']) | 3144 self.ChangeToDepotWorkingDirectory(v['depot']) |
| 3171 info = self.source_control.QueryRevisionInfo(k) | 3145 info = self.source_control.QueryRevisionInfo(k) |
| 3172 culprit_revisions.append((k, info, v['depot'])) | 3146 culprit_revisions.append((k, info, v['depot'])) |
| 3173 os.chdir(cwd) | 3147 os.chdir(cwd) |
| 3174 | 3148 |
| 3175 # Check for any other possible regression ranges | 3149 # Check for any other possible regression ranges. |
| 3176 other_regressions = self._FindOtherRegressions(revision_data_sorted, | 3150 other_regressions = _FindOtherRegressions( |
| 3177 mean_of_bad_runs > mean_of_good_runs) | 3151 revision_data_sorted, mean_of_bad_runs > mean_of_good_runs) |
| 3178 | 3152 |
| 3179 return { | 3153 return { |
| 3180 'first_working_revision': first_working_revision, | 3154 'first_working_revision': first_working_revision, |
| 3181 'last_broken_revision': last_broken_revision, | 3155 'last_broken_revision': last_broken_revision, |
| 3182 'culprit_revisions': culprit_revisions, | 3156 'culprit_revisions': culprit_revisions, |
| 3183 'other_regressions': other_regressions, | 3157 'other_regressions': other_regressions, |
| 3184 'regression_size': regression_size, | 3158 'regression_size': regression_size, |
| 3185 'regression_std_err': regression_std_err, | 3159 'regression_std_err': regression_std_err, |
| 3186 'confidence': confidence, | 3160 'confidence': confidence, |
| 3187 } | 3161 } |
| 3188 | 3162 |
| 3189 def _CheckForWarnings(self, results_dict): | 3163 def _CheckForWarnings(self, results_dict): |
| 3190 if len(results_dict['culprit_revisions']) > 1: | 3164 if len(results_dict['culprit_revisions']) > 1: |
| 3191 self.warnings.append('Due to build errors, regression range could ' | 3165 self.warnings.append('Due to build errors, regression range could ' |
| 3192 'not be narrowed down to a single commit.') | 3166 'not be narrowed down to a single commit.') |
| 3193 if self.opts.repeat_test_count == 1: | 3167 if self.opts.repeat_test_count == 1: |
| 3194 self.warnings.append('Tests were only set to run once. This may ' | 3168 self.warnings.append('Tests were only set to run once. This may ' |
| 3195 'be insufficient to get meaningful results.') | 3169 'be insufficient to get meaningful results.') |
| 3196 if results_dict['confidence'] < 100: | 3170 if results_dict['confidence'] < 100: |
| 3197 if results_dict['confidence']: | 3171 if results_dict['confidence']: |
| 3198 self.warnings.append( | 3172 self.warnings.append( |
| 3199 'Confidence is less than 100%. There could be other candidates ' | 3173 'Confidence is less than 100%. There could be other candidates ' |
| 3200 'for this regression. Try bisecting again with increased ' | 3174 'for this regression. Try bisecting again with increased ' |
| 3201 'repeat_count or on a sub-metric that shows the regression more ' | 3175 'repeat_count or on a sub-metric that shows the regression more ' |
| 3202 'clearly.') | 3176 'clearly.') |
| 3203 else: | 3177 else: |
| 3204 self.warnings.append( | 3178 self.warnings.append( |
| 3205 'Confidence is 0%. Try bisecting again on another platform, with ' | 3179 'Confidence is 0%. Try bisecting again on another platform, with ' |
| 3206 'increased repeat_count or on a sub-metric that shows the ' | 3180 'increased repeat_count or on a sub-metric that shows the ' |
| 3207 'regression more clearly.') | 3181 'regression more clearly.') |
| 3208 | 3182 |
| 3209 def FormatAndPrintResults(self, bisect_results): | 3183 def FormatAndPrintResults(self, bisect_results): |
| 3210 """Prints the results from a bisection run in a readable format. | 3184 """Prints the results from a bisection run in a readable format. |
| 3211 | 3185 |
| 3212 Args | 3186 Args: |
| 3213 bisect_results: The results from a bisection test run. | 3187 bisect_results: The results from a bisection test run. |
| 3214 """ | 3188 """ |
| 3215 revision_data = bisect_results['revision_data'] | 3189 revision_data = bisect_results['revision_data'] |
| 3216 revision_data_sorted = sorted(revision_data.iteritems(), | 3190 revision_data_sorted = sorted(revision_data.iteritems(), |
| 3217 key = lambda x: x[1]['sort']) | 3191 key = lambda x: x[1]['sort']) |
| 3218 results_dict = self._GetResultsDict(revision_data, revision_data_sorted) | 3192 results_dict = self._GetResultsDict(revision_data, revision_data_sorted) |
| 3219 | 3193 |
| 3220 self._CheckForWarnings(results_dict) | 3194 self._CheckForWarnings(results_dict) |
| 3221 | 3195 |
| 3222 if self.opts.output_buildbot_annotations: | 3196 if self.opts.output_buildbot_annotations: |
| (...skipping 27 matching lines...) Expand all Loading... |
| 3250 for culprit in results_dict['culprit_revisions']: | 3224 for culprit in results_dict['culprit_revisions']: |
| 3251 cl, info, depot = culprit | 3225 cl, info, depot = culprit |
| 3252 self._PrintRevisionInfo(cl, info, depot) | 3226 self._PrintRevisionInfo(cl, info, depot) |
| 3253 if results_dict['other_regressions']: | 3227 if results_dict['other_regressions']: |
| 3254 self._PrintOtherRegressions(results_dict['other_regressions'], | 3228 self._PrintOtherRegressions(results_dict['other_regressions'], |
| 3255 revision_data) | 3229 revision_data) |
| 3256 self._PrintTestedCommitsTable(revision_data_sorted, | 3230 self._PrintTestedCommitsTable(revision_data_sorted, |
| 3257 results_dict['first_working_revision'], | 3231 results_dict['first_working_revision'], |
| 3258 results_dict['last_broken_revision'], | 3232 results_dict['last_broken_revision'], |
| 3259 results_dict['confidence']) | 3233 results_dict['confidence']) |
| 3260 self._PrintStepTime(revision_data_sorted) | 3234 _PrintStepTime(revision_data_sorted) |
| 3261 self._PrintReproSteps() | 3235 self._PrintReproSteps() |
| 3262 self._PrintThankYou() | 3236 self._PrintThankYou() |
| 3263 if self.opts.output_buildbot_annotations: | 3237 if self.opts.output_buildbot_annotations: |
| 3264 bisect_utils.OutputAnnotationStepClosed() | 3238 bisect_utils.OutputAnnotationStepClosed() |
| 3265 | 3239 |
| 3240 def _PrintBanner(self, results_dict): |
| 3241 if self._IsBisectModeReturnCode(): |
| 3242 metrics = 'N/A' |
| 3243 change = 'Yes' |
| 3244 else: |
| 3245 metrics = '/'.join(self.opts.metric) |
| 3246 change = '%.02f%% (+/-%.02f%%)' % ( |
| 3247 results_dict['regression_size'], results_dict['regression_std_err']) |
| 3266 | 3248 |
| 3267 def IsPlatformSupported(opts): | 3249 if results_dict['culprit_revisions'] and results_dict['confidence']: |
| 3250 status = self._ConfidenceLevelStatus(results_dict) |
| 3251 else: |
| 3252 status = 'Failure, could not reproduce.' |
| 3253 change = 'Bisect could not reproduce a change.' |
| 3254 |
| 3255 print RESULTS_BANNER % { |
| 3256 'status': status, |
| 3257 'command': self.opts.command, |
| 3258 'metrics': metrics, |
| 3259 'change': change, |
| 3260 'confidence': results_dict['confidence'], |
| 3261 } |
| 3262 |
| 3263 def _PrintWarnings(self): |
| 3264 """Prints a list of warning strings if there are any.""" |
| 3265 if not self.warnings: |
| 3266 return |
| 3267 print |
| 3268 print 'WARNINGS:' |
| 3269 for w in set(self.warnings): |
| 3270 print ' ! %s' % w |
| 3271 |
| 3272 |
| 3273 def _IsPlatformSupported(): |
| 3268 """Checks that this platform and build system are supported. | 3274 """Checks that this platform and build system are supported. |
| 3269 | 3275 |
| 3270 Args: | 3276 Args: |
| 3271 opts: The options parsed from the command line. | 3277 opts: The options parsed from the command line. |
| 3272 | 3278 |
| 3273 Returns: | 3279 Returns: |
| 3274 True if the platform and build system are supported. | 3280 True if the platform and build system are supported. |
| 3275 """ | 3281 """ |
| 3276 # Haven't tested the script out on any other platforms yet. | 3282 # Haven't tested the script out on any other platforms yet. |
| 3277 supported = ['posix', 'nt'] | 3283 supported = ['posix', 'nt'] |
| (...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3335 self.debug_ignore_build = None | 3341 self.debug_ignore_build = None |
| 3336 self.debug_ignore_sync = None | 3342 self.debug_ignore_sync = None |
| 3337 self.debug_ignore_perf_test = None | 3343 self.debug_ignore_perf_test = None |
| 3338 self.gs_bucket = None | 3344 self.gs_bucket = None |
| 3339 self.target_arch = 'ia32' | 3345 self.target_arch = 'ia32' |
| 3340 self.target_build_type = 'Release' | 3346 self.target_build_type = 'Release' |
| 3341 self.builder_host = None | 3347 self.builder_host = None |
| 3342 self.builder_port = None | 3348 self.builder_port = None |
| 3343 self.bisect_mode = BISECT_MODE_MEAN | 3349 self.bisect_mode = BISECT_MODE_MEAN |
| 3344 | 3350 |
| 3345 def _CreateCommandLineParser(self): | 3351 @staticmethod |
| 3352 def _CreateCommandLineParser(): |
| 3346 """Creates a parser with bisect options. | 3353 """Creates a parser with bisect options. |
| 3347 | 3354 |
| 3348 Returns: | 3355 Returns: |
| 3349 An instance of optparse.OptionParser. | 3356 An instance of optparse.OptionParser. |
| 3350 """ | 3357 """ |
| 3351 usage = ('%prog [options] [-- chromium-options]\n' | 3358 usage = ('%prog [options] [-- chromium-options]\n' |
| 3352 'Perform binary search on revision history to find a minimal ' | 3359 'Perform binary search on revision history to find a minimal ' |
| 3353 'range of revisions where a peformance metric regressed.\n') | 3360 'range of revisions where a peformance metric regressed.\n') |
| 3354 | 3361 |
| 3355 parser = optparse.OptionParser(usage=usage) | 3362 parser = optparse.OptionParser(usage=usage) |
| (...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3422 group.add_option('--target_platform', | 3429 group.add_option('--target_platform', |
| 3423 type='choice', | 3430 type='choice', |
| 3424 choices=['chromium', 'cros', 'android', 'android-chrome'], | 3431 choices=['chromium', 'cros', 'android', 'android-chrome'], |
| 3425 default='chromium', | 3432 default='chromium', |
| 3426 help='The target platform. Choices are "chromium" ' | 3433 help='The target platform. Choices are "chromium" ' |
| 3427 '(current platform), "cros", or "android". If you ' | 3434 '(current platform), "cros", or "android". If you ' |
| 3428 'specify something other than "chromium", you must be ' | 3435 'specify something other than "chromium", you must be ' |
| 3429 'properly set up to build that platform.') | 3436 'properly set up to build that platform.') |
| 3430 group.add_option('--no_custom_deps', | 3437 group.add_option('--no_custom_deps', |
| 3431 dest='no_custom_deps', | 3438 dest='no_custom_deps', |
| 3432 action="store_true", | 3439 action='store_true', |
| 3433 default=False, | 3440 default=False, |
| 3434 help='Run the script with custom_deps or not.') | 3441 help='Run the script with custom_deps or not.') |
| 3435 group.add_option('--extra_src', | 3442 group.add_option('--extra_src', |
| 3436 type='str', | 3443 type='str', |
| 3437 help='Path to a script which can be used to modify ' | 3444 help='Path to a script which can be used to modify ' |
| 3438 'the bisect script\'s behavior.') | 3445 'the bisect script\'s behavior.') |
| 3439 group.add_option('--cros_board', | 3446 group.add_option('--cros_board', |
| 3440 type='str', | 3447 type='str', |
| 3441 help='The cros board type to build.') | 3448 help='The cros board type to build.') |
| 3442 group.add_option('--cros_remote_ip', | 3449 group.add_option('--cros_remote_ip', |
| 3443 type='str', | 3450 type='str', |
| 3444 help='The remote machine to image to.') | 3451 help='The remote machine to image to.') |
| 3445 group.add_option('--use_goma', | 3452 group.add_option('--use_goma', |
| 3446 action="store_true", | 3453 action='store_true', |
| 3447 help='Add a bunch of extra threads for goma, and enable ' | 3454 help='Add a bunch of extra threads for goma, and enable ' |
| 3448 'goma') | 3455 'goma') |
| 3449 group.add_option('--goma_dir', | 3456 group.add_option('--goma_dir', |
| 3450 help='Path to goma tools (or system default if not ' | 3457 help='Path to goma tools (or system default if not ' |
| 3451 'specified).') | 3458 'specified).') |
| 3452 group.add_option('--output_buildbot_annotations', | 3459 group.add_option('--output_buildbot_annotations', |
| 3453 action="store_true", | 3460 action='store_true', |
| 3454 help='Add extra annotation output for buildbot.') | 3461 help='Add extra annotation output for buildbot.') |
| 3455 group.add_option('--gs_bucket', | 3462 group.add_option('--gs_bucket', |
| 3456 default='', | 3463 default='', |
| 3457 dest='gs_bucket', | 3464 dest='gs_bucket', |
| 3458 type='str', | 3465 type='str', |
| 3459 help=('Name of Google Storage bucket to upload or ' | 3466 help=('Name of Google Storage bucket to upload or ' |
| 3460 'download build. e.g., chrome-perf')) | 3467 'download build. e.g., chrome-perf')) |
| 3461 group.add_option('--target_arch', | 3468 group.add_option('--target_arch', |
| 3462 type='choice', | 3469 type='choice', |
| 3463 choices=['ia32', 'x64', 'arm'], | 3470 choices=['ia32', 'x64', 'arm'], |
| (...skipping 14 matching lines...) Expand all Loading... |
| 3478 ' try job request.')) | 3485 ' try job request.')) |
| 3479 group.add_option('--builder_port', | 3486 group.add_option('--builder_port', |
| 3480 dest='builder_port', | 3487 dest='builder_port', |
| 3481 type='int', | 3488 type='int', |
| 3482 help=('HTTP port of the server to produce build by posting' | 3489 help=('HTTP port of the server to produce build by posting' |
| 3483 ' try job request.')) | 3490 ' try job request.')) |
| 3484 parser.add_option_group(group) | 3491 parser.add_option_group(group) |
| 3485 | 3492 |
| 3486 group = optparse.OptionGroup(parser, 'Debug options') | 3493 group = optparse.OptionGroup(parser, 'Debug options') |
| 3487 group.add_option('--debug_ignore_build', | 3494 group.add_option('--debug_ignore_build', |
| 3488 action="store_true", | 3495 action='store_true', |
| 3489 help='DEBUG: Don\'t perform builds.') | 3496 help='DEBUG: Don\'t perform builds.') |
| 3490 group.add_option('--debug_ignore_sync', | 3497 group.add_option('--debug_ignore_sync', |
| 3491 action="store_true", | 3498 action='store_true', |
| 3492 help='DEBUG: Don\'t perform syncs.') | 3499 help='DEBUG: Don\'t perform syncs.') |
| 3493 group.add_option('--debug_ignore_perf_test', | 3500 group.add_option('--debug_ignore_perf_test', |
| 3494 action="store_true", | 3501 action='store_true', |
| 3495 help='DEBUG: Don\'t perform performance tests.') | 3502 help='DEBUG: Don\'t perform performance tests.') |
| 3496 parser.add_option_group(group) | 3503 parser.add_option_group(group) |
| 3497 return parser | 3504 return parser |
| 3498 | 3505 |
| 3499 def ParseCommandLine(self): | 3506 def ParseCommandLine(self): |
| 3500 """Parses the command line for bisect options.""" | 3507 """Parses the command line for bisect options.""" |
| 3501 parser = self._CreateCommandLineParser() | 3508 parser = self._CreateCommandLineParser() |
| 3502 (opts, _) = parser.parse_args() | 3509 opts, _ = parser.parse_args() |
| 3503 | 3510 |
| 3504 try: | 3511 try: |
| 3505 if not opts.command: | 3512 if not opts.command: |
| 3506 raise RuntimeError('missing required parameter: --command') | 3513 raise RuntimeError('missing required parameter: --command') |
| 3507 | 3514 |
| 3508 if not opts.good_revision: | 3515 if not opts.good_revision: |
| 3509 raise RuntimeError('missing required parameter: --good_revision') | 3516 raise RuntimeError('missing required parameter: --good_revision') |
| 3510 | 3517 |
| 3511 if not opts.bad_revision: | 3518 if not opts.bad_revision: |
| 3512 raise RuntimeError('missing required parameter: --bad_revision') | 3519 raise RuntimeError('missing required parameter: --bad_revision') |
| (...skipping 21 matching lines...) Expand all Loading... |
| 3534 | 3541 |
| 3535 if not opts.cros_remote_ip: | 3542 if not opts.cros_remote_ip: |
| 3536 raise RuntimeError('missing required parameter: --cros_remote_ip') | 3543 raise RuntimeError('missing required parameter: --cros_remote_ip') |
| 3537 | 3544 |
| 3538 if not opts.working_directory: | 3545 if not opts.working_directory: |
| 3539 raise RuntimeError('missing required parameter: --working_directory') | 3546 raise RuntimeError('missing required parameter: --working_directory') |
| 3540 | 3547 |
| 3541 metric_values = opts.metric.split('/') | 3548 metric_values = opts.metric.split('/') |
| 3542 if (len(metric_values) != 2 and | 3549 if (len(metric_values) != 2 and |
| 3543 opts.bisect_mode != BISECT_MODE_RETURN_CODE): | 3550 opts.bisect_mode != BISECT_MODE_RETURN_CODE): |
| 3544 raise RuntimeError("Invalid metric specified: [%s]" % opts.metric) | 3551 raise RuntimeError('Invalid metric specified: [%s]' % opts.metric) |
| 3545 | 3552 |
| 3546 opts.metric = metric_values | 3553 opts.metric = metric_values |
| 3547 opts.repeat_test_count = min(max(opts.repeat_test_count, 1), 100) | 3554 opts.repeat_test_count = min(max(opts.repeat_test_count, 1), 100) |
| 3548 opts.max_time_minutes = min(max(opts.max_time_minutes, 1), 60) | 3555 opts.max_time_minutes = min(max(opts.max_time_minutes, 1), 60) |
| 3549 opts.truncate_percent = min(max(opts.truncate_percent, 0), 25) | 3556 opts.truncate_percent = min(max(opts.truncate_percent, 0), 25) |
| 3550 opts.truncate_percent = opts.truncate_percent / 100.0 | 3557 opts.truncate_percent = opts.truncate_percent / 100.0 |
| 3551 | 3558 |
| 3552 for k, v in opts.__dict__.iteritems(): | 3559 for k, v in opts.__dict__.iteritems(): |
| 3553 assert hasattr(self, k), "Invalid %s attribute in BisectOptions." % k | 3560 assert hasattr(self, k), 'Invalid %s attribute in BisectOptions.' % k |
| 3554 setattr(self, k, v) | 3561 setattr(self, k, v) |
| 3555 except RuntimeError, e: | 3562 except RuntimeError, e: |
| 3556 output_string = StringIO.StringIO() | 3563 output_string = StringIO.StringIO() |
| 3557 parser.print_help(file=output_string) | 3564 parser.print_help(file=output_string) |
| 3558 error_message = '%s\n\n%s' % (e.message, output_string.getvalue()) | 3565 error_message = '%s\n\n%s' % (e.message, output_string.getvalue()) |
| 3559 output_string.close() | 3566 output_string.close() |
| 3560 raise RuntimeError(error_message) | 3567 raise RuntimeError(error_message) |
| 3561 | 3568 |
| 3562 @staticmethod | 3569 @staticmethod |
| 3563 def FromDict(values): | 3570 def FromDict(values): |
| 3564 """Creates an instance of BisectOptions with the values parsed from a | 3571 """Creates an instance of BisectOptions with the values parsed from a |
| 3565 .cfg file. | 3572 .cfg file. |
| 3566 | 3573 |
| 3567 Args: | 3574 Args: |
| 3568 values: a dict containing options to set. | 3575 values: a dict containing options to set. |
| 3569 | 3576 |
| 3570 Returns: | 3577 Returns: |
| 3571 An instance of BisectOptions. | 3578 An instance of BisectOptions. |
| 3572 """ | 3579 """ |
| 3573 opts = BisectOptions() | 3580 opts = BisectOptions() |
| 3574 for k, v in values.iteritems(): | 3581 for k, v in values.iteritems(): |
| 3575 assert hasattr(opts, k), 'Invalid %s attribute in '\ | 3582 assert hasattr(opts, k), 'Invalid %s attribute in BisectOptions.' % k |
| 3576 'BisectOptions.' % k | |
| 3577 setattr(opts, k, v) | 3583 setattr(opts, k, v) |
| 3578 | 3584 |
| 3579 metric_values = opts.metric.split('/') | 3585 metric_values = opts.metric.split('/') |
| 3580 if len(metric_values) != 2: | 3586 if len(metric_values) != 2: |
| 3581 raise RuntimeError("Invalid metric specified: [%s]" % opts.metric) | 3587 raise RuntimeError('Invalid metric specified: [%s]' % opts.metric) |
| 3582 | 3588 |
| 3583 opts.metric = metric_values | 3589 opts.metric = metric_values |
| 3584 opts.repeat_test_count = min(max(opts.repeat_test_count, 1), 100) | 3590 opts.repeat_test_count = min(max(opts.repeat_test_count, 1), 100) |
| 3585 opts.max_time_minutes = min(max(opts.max_time_minutes, 1), 60) | 3591 opts.max_time_minutes = min(max(opts.max_time_minutes, 1), 60) |
| 3586 opts.truncate_percent = min(max(opts.truncate_percent, 0), 25) | 3592 opts.truncate_percent = min(max(opts.truncate_percent, 0), 25) |
| 3587 opts.truncate_percent = opts.truncate_percent / 100.0 | 3593 opts.truncate_percent = opts.truncate_percent / 100.0 |
| 3588 | 3594 |
| 3589 return opts | 3595 return opts |
| 3590 | 3596 |
| 3591 | 3597 |
| 3592 def main(): | 3598 def main(): |
| 3593 | 3599 |
| 3594 try: | 3600 try: |
| 3595 opts = BisectOptions() | 3601 opts = BisectOptions() |
| 3596 opts.ParseCommandLine() | 3602 opts.ParseCommandLine() |
| 3597 | 3603 |
| 3598 if opts.extra_src: | 3604 if opts.extra_src: |
| 3599 extra_src = bisect_utils.LoadExtraSrc(opts.extra_src) | 3605 extra_src = bisect_utils.LoadExtraSrc(opts.extra_src) |
| 3600 if not extra_src: | 3606 if not extra_src: |
| 3601 raise RuntimeError("Invalid or missing --extra_src.") | 3607 raise RuntimeError('Invalid or missing --extra_src.') |
| 3602 _AddAdditionalDepotInfo(extra_src.GetAdditionalDepotInfo()) | 3608 _AddAdditionalDepotInfo(extra_src.GetAdditionalDepotInfo()) |
| 3603 | 3609 |
| 3604 if opts.working_directory: | 3610 if opts.working_directory: |
| 3605 custom_deps = bisect_utils.DEFAULT_GCLIENT_CUSTOM_DEPS | 3611 custom_deps = bisect_utils.DEFAULT_GCLIENT_CUSTOM_DEPS |
| 3606 if opts.no_custom_deps: | 3612 if opts.no_custom_deps: |
| 3607 custom_deps = None | 3613 custom_deps = None |
| 3608 bisect_utils.CreateBisectDirectoryAndSetupDepot(opts, custom_deps) | 3614 bisect_utils.CreateBisectDirectoryAndSetupDepot(opts, custom_deps) |
| 3609 | 3615 |
| 3610 os.chdir(os.path.join(os.getcwd(), 'src')) | 3616 os.chdir(os.path.join(os.getcwd(), 'src')) |
| 3611 | 3617 |
| 3612 if not RemoveBuildFiles(opts.target_build_type): | 3618 if not RemoveBuildFiles(opts.target_build_type): |
| 3613 raise RuntimeError('Something went wrong removing the build files.') | 3619 raise RuntimeError('Something went wrong removing the build files.') |
| 3614 | 3620 |
| 3615 if not IsPlatformSupported(opts): | 3621 if not _IsPlatformSupported(): |
| 3616 raise RuntimeError("Sorry, this platform isn't supported yet.") | 3622 raise RuntimeError('Sorry, this platform isn\'t supported yet.') |
| 3617 | 3623 |
| 3618 # Check what source control method is being used, and create a | 3624 # Check what source control method is being used, and create a |
| 3619 # SourceControl object if possible. | 3625 # SourceControl object if possible. |
| 3620 source_control = source_control_module.DetermineAndCreateSourceControl(opts) | 3626 source_control = source_control_module.DetermineAndCreateSourceControl(opts) |
| 3621 | 3627 |
| 3622 if not source_control: | 3628 if not source_control: |
| 3623 raise RuntimeError("Sorry, only the git workflow is supported at the " | 3629 raise RuntimeError( |
| 3624 "moment.") | 3630 'Sorry, only the git workflow is supported at the moment.') |
| 3625 | 3631 |
| 3626 # gClient sync seems to fail if you're not in master branch. | 3632 # gClient sync seems to fail if you're not in master branch. |
| 3627 if (not source_control.IsInProperBranch() and | 3633 if (not source_control.IsInProperBranch() and |
| 3628 not opts.debug_ignore_sync and | 3634 not opts.debug_ignore_sync and |
| 3629 not opts.working_directory): | 3635 not opts.working_directory): |
| 3630 raise RuntimeError("You must switch to master branch to run bisection.") | 3636 raise RuntimeError('You must switch to master branch to run bisection.') |
| 3631 bisect_test = BisectPerformanceMetrics(source_control, opts) | 3637 bisect_test = BisectPerformanceMetrics(source_control, opts) |
| 3632 try: | 3638 try: |
| 3633 bisect_results = bisect_test.Run(opts.command, | 3639 bisect_results = bisect_test.Run(opts.command, |
| 3634 opts.bad_revision, | 3640 opts.bad_revision, |
| 3635 opts.good_revision, | 3641 opts.good_revision, |
| 3636 opts.metric) | 3642 opts.metric) |
| 3637 if bisect_results['error']: | 3643 if bisect_results['error']: |
| 3638 raise RuntimeError(bisect_results['error']) | 3644 raise RuntimeError(bisect_results['error']) |
| 3639 bisect_test.FormatAndPrintResults(bisect_results) | 3645 bisect_test.FormatAndPrintResults(bisect_results) |
| 3640 return 0 | 3646 return 0 |
| 3641 finally: | 3647 finally: |
| 3642 bisect_test.PerformCleanup() | 3648 bisect_test.PerformCleanup() |
| 3643 except RuntimeError, e: | 3649 except RuntimeError, e: |
| 3644 if opts.output_buildbot_annotations: | 3650 if opts.output_buildbot_annotations: |
| 3645 # The perf dashboard scrapes the "results" step in order to comment on | 3651 # The perf dashboard scrapes the "results" step in order to comment on |
| 3646 # bugs. If you change this, please update the perf dashboard as well. | 3652 # bugs. If you change this, please update the perf dashboard as well. |
| 3647 bisect_utils.OutputAnnotationStepStart('Results') | 3653 bisect_utils.OutputAnnotationStepStart('Results') |
| 3648 print 'Error: %s' % e.message | 3654 print 'Error: %s' % e.message |
| 3649 if opts.output_buildbot_annotations: | 3655 if opts.output_buildbot_annotations: |
| 3650 bisect_utils.OutputAnnotationStepClosed() | 3656 bisect_utils.OutputAnnotationStepClosed() |
| 3651 return 1 | 3657 return 1 |
| 3652 | 3658 |
| 3653 | 3659 |
| 3654 if __name__ == '__main__': | 3660 if __name__ == '__main__': |
| 3655 sys.exit(main()) | 3661 sys.exit(main()) |
| OLD | NEW |