Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(19)

Side by Side Diff: tools/bisect-perf-regression.py

Issue 564663002: Move bisect-perf-regression.py into auto_bisect directory. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Add telemetry to PYTHONPATH for PRESUBMIT.py. Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/auto_bisect/source_control.py ('k') | tools/bisect-perf-regression_test.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 """Performance Test Bisect Tool
7
8 This script bisects a series of changelists using binary search. It starts at
9 a bad revision where a performance metric has regressed, and asks for a last
10 known-good revision. It will then binary search across this revision range by
11 syncing, building, and running a performance test. If the change is
12 suspected to occur as a result of WebKit/V8 changes, the script will
13 further bisect changes to those depots and attempt to narrow down the revision
14 range.
15
16 Example usage using SVN revisions:
17
18 ./tools/bisect-perf-regression.py -c\
19 "out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\
20 -g 168222 -b 168232 -m shutdown/simple-user-quit
21
22 Be aware that if you're using the git workflow and specify an SVN revision,
23 the script will attempt to find the git SHA1 where SVN changes up to that
24 revision were merged in.
25
26 Example usage using git hashes:
27
28 ./tools/bisect-perf-regression.py -c\
29 "out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\
30 -g 1f6e67861535121c5c819c16a666f2436c207e7b\
31 -b b732f23b4f81c382db0b23b9035f3dadc7d925bb\
32 -m shutdown/simple-user-quit
33 """
34
35 import copy
36 import datetime
37 import errno
38 import hashlib
39 import math
40 import optparse
41 import os
42 import re
43 import shlex
44 import shutil
45 import StringIO
46 import sys
47 import time
48 import zipfile
49
50 sys.path.append(os.path.join(os.path.dirname(__file__), 'telemetry'))
51
52 from auto_bisect import bisect_utils
53 from auto_bisect import builder
54 from auto_bisect import math_utils
55 from auto_bisect import request_build
56 from auto_bisect import source_control as source_control_module
57 from auto_bisect import ttest
58 from telemetry.util import cloud_storage
59
60 # Below is the map of "depot" names to information about each depot. Each depot
61 # is a repository, and in the process of bisecting, revision ranges in these
62 # repositories may also be bisected.
63 #
64 # Each depot information dictionary may contain:
65 # src: Path to the working directory.
66 # recurse: True if this repository will get bisected.
67 # depends: A list of other repositories that are actually part of the same
68 # repository in svn. If the repository has any dependent repositories
69 # (e.g. skia/src needs skia/include and skia/gyp to be updated), then
70 # they are specified here.
71 # svn: URL of SVN repository. Needed for git workflow to resolve hashes to
72 # SVN revisions.
73 # from: Parent depot that must be bisected before this is bisected.
74 # deps_var: Key name in vars variable in DEPS file that has revision
75 # information.
76 DEPOT_DEPS_NAME = {
77 'chromium': {
78 'src': 'src',
79 'recurse': True,
80 'depends': None,
81 'from': ['cros', 'android-chrome'],
82 'viewvc':
83 'http://src.chromium.org/viewvc/chrome?view=revision&revision=',
84 'deps_var': 'chromium_rev'
85 },
86 'webkit': {
87 'src': 'src/third_party/WebKit',
88 'recurse': True,
89 'depends': None,
90 'from': ['chromium'],
91 'viewvc':
92 'http://src.chromium.org/viewvc/blink?view=revision&revision=',
93 'deps_var': 'webkit_revision'
94 },
95 'angle': {
96 'src': 'src/third_party/angle',
97 'src_old': 'src/third_party/angle_dx11',
98 'recurse': True,
99 'depends': None,
100 'from': ['chromium'],
101 'platform': 'nt',
102 'deps_var': 'angle_revision'
103 },
104 'v8': {
105 'src': 'src/v8',
106 'recurse': True,
107 'depends': None,
108 'from': ['chromium'],
109 'custom_deps': bisect_utils.GCLIENT_CUSTOM_DEPS_V8,
110 'viewvc': 'https://code.google.com/p/v8/source/detail?r=',
111 'deps_var': 'v8_revision'
112 },
113 'v8_bleeding_edge': {
114 'src': 'src/v8_bleeding_edge',
115 'recurse': True,
116 'depends': None,
117 'svn': 'https://v8.googlecode.com/svn/branches/bleeding_edge',
118 'from': ['v8'],
119 'viewvc': 'https://code.google.com/p/v8/source/detail?r=',
120 'deps_var': 'v8_revision'
121 },
122 'skia/src': {
123 'src': 'src/third_party/skia/src',
124 'recurse': True,
125 'svn': 'http://skia.googlecode.com/svn/trunk/src',
126 'depends': ['skia/include', 'skia/gyp'],
127 'from': ['chromium'],
128 'viewvc': 'https://code.google.com/p/skia/source/detail?r=',
129 'deps_var': 'skia_revision'
130 },
131 'skia/include': {
132 'src': 'src/third_party/skia/include',
133 'recurse': False,
134 'svn': 'http://skia.googlecode.com/svn/trunk/include',
135 'depends': None,
136 'from': ['chromium'],
137 'viewvc': 'https://code.google.com/p/skia/source/detail?r=',
138 'deps_var': 'None'
139 },
140 'skia/gyp': {
141 'src': 'src/third_party/skia/gyp',
142 'recurse': False,
143 'svn': 'http://skia.googlecode.com/svn/trunk/gyp',
144 'depends': None,
145 'from': ['chromium'],
146 'viewvc': 'https://code.google.com/p/skia/source/detail?r=',
147 'deps_var': 'None'
148 }
149 }
150
151 DEPOT_NAMES = DEPOT_DEPS_NAME.keys()
152
153 CROS_CHROMEOS_PATTERN = 'chromeos-base/chromeos-chrome'
154
155 # Possible return values from BisectPerformanceMetrics.RunTest.
156 BUILD_RESULT_SUCCEED = 0
157 BUILD_RESULT_FAIL = 1
158 BUILD_RESULT_SKIPPED = 2
159
160 # Maximum time in seconds to wait after posting build request to the try server.
161 # TODO: Change these values based on the actual time taken by buildbots on
162 # the try server.
163 MAX_MAC_BUILD_TIME = 14400
164 MAX_WIN_BUILD_TIME = 14400
165 MAX_LINUX_BUILD_TIME = 14400
166
167 # The confidence percentage at which confidence can be consider "high".
168 HIGH_CONFIDENCE = 95
169
170 # Patch template to add a new file, DEPS.sha under src folder.
171 # This file contains SHA1 value of the DEPS changes made while bisecting
172 # dependency repositories. This patch send along with DEPS patch to try server.
173 # When a build requested is posted with a patch, bisect builders on try server,
174 # once build is produced, it reads SHA value from this file and appends it
175 # to build archive filename.
176 DEPS_SHA_PATCH = """diff --git src/DEPS.sha src/DEPS.sha
177 new file mode 100644
178 --- /dev/null
179 +++ src/DEPS.sha
180 @@ -0,0 +1 @@
181 +%(deps_sha)s
182 """
183
184 # The possible values of the --bisect_mode flag, which determines what to
185 # use when classifying a revision as "good" or "bad".
186 BISECT_MODE_MEAN = 'mean'
187 BISECT_MODE_STD_DEV = 'std_dev'
188 BISECT_MODE_RETURN_CODE = 'return_code'
189
190 # The perf dashboard looks for a string like "Estimated Confidence: 95%"
191 # to decide whether or not to cc the author(s). If you change this, please
192 # update the perf dashboard as well.
193 RESULTS_BANNER = """
194 ===== BISECT JOB RESULTS =====
195 Status: %(status)s
196
197 Test Command: %(command)s
198 Test Metric: %(metrics)s
199 Relative Change: %(change)s
200 Estimated Confidence: %(confidence).02f%%"""
201
202 # The perf dashboard specifically looks for the string
203 # "Author : " to parse out who to cc on a bug. If you change the
204 # formatting here, please update the perf dashboard as well.
205 RESULTS_REVISION_INFO = """
206 ===== SUSPECTED CL(s) =====
207 Subject : %(subject)s
208 Author : %(author)s%(email_info)s%(commit_info)s
209 Commit : %(cl)s
210 Date : %(cl_date)s"""
211
212 REPRO_STEPS_LOCAL = """
213 ==== INSTRUCTIONS TO REPRODUCE ====
214 To run locally:
215 - Use the test command given under 'BISECT JOB RESULTS' above.
216 - Consider using a profiler. Pass --profiler=list to list available profilers.
217 """
218
219 REPRO_STEPS_TRYJOB = """
220 To reproduce on a performance try bot:
221 1. Edit run-perf-test.cfg
222 2. Upload your patch with: $ git cl upload --bypass-hooks
223 3. Send to the try server: $ git cl try -m tryserver.chromium.perf -b <bot>
224
225 Notes:
226 a) Follow the in-file instructions in run-perf-test.cfg.
227 b) run-perf-test.cfg is under tools/ or under third_party/WebKit/Tools.
228 c) Do your edits preferably under a new git branch.
229 d) --browser=release and --browser=android-chromium-testshell are supported
230 depending on the platform (desktop|android).
231 e) Strip any src/ directories from the head of relative path names.
232 f) Make sure to use the appropriate bot on step 3.
233
234 For more details please visit
235 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots"""
236
237 REPRO_STEPS_TRYJOB_TELEMETRY = """
238 To reproduce on a performance try bot:
239 %(command)s
240 (Where <bot-name> comes from tools/perf/run_benchmark --browser=list)
241
242 For more details please visit
243 https://sites.google.com/a/chromium.org/dev/developers/performance-try-bots
244 """
245
246 RESULTS_THANKYOU = """
247 ===== THANK YOU FOR CHOOSING BISECT AIRLINES =====
248 Visit http://www.chromium.org/developers/core-principles for Chrome's policy
249 on perf regressions.
250 Contact chrome-perf-dashboard-team with any questions or suggestions about
251 bisecting.
252 . .-----.
253 . .---. \ \==)
254 . |PERF\ \ \\
255 . | ---------'-------'-----------.
256 . . 0 0 0 0 0 0 0 0 0 0 0 0 0 0 |_`-.
257 . \_____________.-------._______________)
258 . / /
259 . / /
260 . / /==)
261 . ._____."""
262
263
264 def _AddAdditionalDepotInfo(depot_info):
265 """Adds additional depot info to the global depot variables."""
266 global DEPOT_DEPS_NAME
267 global DEPOT_NAMES
268 DEPOT_DEPS_NAME = dict(DEPOT_DEPS_NAME.items() + depot_info.items())
269 DEPOT_NAMES = DEPOT_DEPS_NAME.keys()
270
271
272 def ConfidenceScore(good_results_lists, bad_results_lists):
273 """Calculates a confidence score.
274
275 This score is a percentage which represents our degree of confidence in the
276 proposition that the good results and bad results are distinct groups, and
277 their differences aren't due to chance alone.
278
279
280 Args:
281 good_results_lists: A list of lists of "good" result numbers.
282 bad_results_lists: A list of lists of "bad" result numbers.
283
284 Returns:
285 A number in the range [0, 100].
286 """
287 # If there's only one item in either list, this means only one revision was
288 # classified good or bad; this isn't good enough evidence to make a decision.
289 # If an empty list was passed, that also implies zero confidence.
290 if len(good_results_lists) <= 1 or len(bad_results_lists) <= 1:
291 return 0.0
292
293 # Flatten the lists of results lists.
294 sample1 = sum(good_results_lists, [])
295 sample2 = sum(bad_results_lists, [])
296
297 # If there were only empty lists in either of the lists (this is unexpected
298 # and normally shouldn't happen), then we also want to return 0.
299 if not sample1 or not sample2:
300 return 0.0
301
302 # The p-value is approximately the probability of obtaining the given set
303 # of good and bad values just by chance.
304 _, _, p_value = ttest.WelchsTTest(sample1, sample2)
305 return 100.0 * (1.0 - p_value)
306
307
308 def GetSHA1HexDigest(contents):
309 """Returns SHA1 hex digest of the given string."""
310 return hashlib.sha1(contents).hexdigest()
311
312
313 def GetZipFileName(build_revision=None, target_arch='ia32', patch_sha=None):
314 """Gets the archive file name for the given revision."""
315 def PlatformName():
316 """Return a string to be used in paths for the platform."""
317 if bisect_utils.IsWindowsHost():
318 # Build archive for x64 is still stored with the "win32" suffix.
319 # See chromium_utils.PlatformName().
320 if bisect_utils.Is64BitWindows() and target_arch == 'x64':
321 return 'win32'
322 return 'win32'
323 if bisect_utils.IsLinuxHost():
324 # Android builds are also archived with the "full-build-linux prefix.
325 return 'linux'
326 if bisect_utils.IsMacHost():
327 return 'mac'
328 raise NotImplementedError('Unknown platform "%s".' % sys.platform)
329
330 base_name = 'full-build-%s' % PlatformName()
331 if not build_revision:
332 return base_name
333 if patch_sha:
334 build_revision = '%s_%s' % (build_revision , patch_sha)
335 return '%s_%s.zip' % (base_name, build_revision)
336
337
338 def GetRemoteBuildPath(build_revision, target_platform='chromium',
339 target_arch='ia32', patch_sha=None):
340 """Returns the URL to download the build from."""
341 def GetGSRootFolderName(target_platform):
342 """Returns the Google Cloud Storage root folder name."""
343 if bisect_utils.IsWindowsHost():
344 if bisect_utils.Is64BitWindows() and target_arch == 'x64':
345 return 'Win x64 Builder'
346 return 'Win Builder'
347 if bisect_utils.IsLinuxHost():
348 if target_platform == 'android':
349 return 'android_perf_rel'
350 return 'Linux Builder'
351 if bisect_utils.IsMacHost():
352 return 'Mac Builder'
353 raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
354
355 base_filename = GetZipFileName(
356 build_revision, target_arch, patch_sha)
357 builder_folder = GetGSRootFolderName(target_platform)
358 return '%s/%s' % (builder_folder, base_filename)
359
360
361 def FetchFromCloudStorage(bucket_name, source_path, destination_path):
362 """Fetches file(s) from the Google Cloud Storage.
363
364 Args:
365 bucket_name: Google Storage bucket name.
366 source_path: Source file path.
367 destination_path: Destination file path.
368
369 Returns:
370 Downloaded file path if exists, otherwise None.
371 """
372 target_file = os.path.join(destination_path, os.path.basename(source_path))
373 try:
374 if cloud_storage.Exists(bucket_name, source_path):
375 print 'Fetching file from gs//%s/%s ...' % (bucket_name, source_path)
376 cloud_storage.Get(bucket_name, source_path, destination_path)
377 if os.path.exists(target_file):
378 return target_file
379 else:
380 print ('File gs://%s/%s not found in cloud storage.' % (
381 bucket_name, source_path))
382 except Exception as e:
383 print 'Something went wrong while fetching file from cloud: %s' % e
384 if os.path.exists(target_file):
385 os.remove(target_file)
386 return None
387
388
389 # This is copied from build/scripts/common/chromium_utils.py.
390 def MaybeMakeDirectory(*path):
391 """Creates an entire path, if it doesn't already exist."""
392 file_path = os.path.join(*path)
393 try:
394 os.makedirs(file_path)
395 except OSError as e:
396 if e.errno != errno.EEXIST:
397 return False
398 return True
399
400
401 # This was copied from build/scripts/common/chromium_utils.py.
402 def ExtractZip(filename, output_dir, verbose=True):
403 """ Extract the zip archive in the output directory."""
404 MaybeMakeDirectory(output_dir)
405
406 # On Linux and Mac, we use the unzip command as it will
407 # handle links and file bits (executable), which is much
408 # easier then trying to do that with ZipInfo options.
409 #
410 # The Mac Version of unzip unfortunately does not support Zip64, whereas
411 # the python module does, so we have to fall back to the python zip module
412 # on Mac if the file size is greater than 4GB.
413 #
414 # On Windows, try to use 7z if it is installed, otherwise fall back to python
415 # zip module and pray we don't have files larger than 512MB to unzip.
416 unzip_cmd = None
417 if ((bisect_utils.IsMacHost()
418 and os.path.getsize(filename) < 4 * 1024 * 1024 * 1024)
419 or bisect_utils.IsLinuxHost()):
420 unzip_cmd = ['unzip', '-o']
421 elif (bisect_utils.IsWindowsHost()
422 and os.path.exists('C:\\Program Files\\7-Zip\\7z.exe')):
423 unzip_cmd = ['C:\\Program Files\\7-Zip\\7z.exe', 'x', '-y']
424
425 if unzip_cmd:
426 # Make sure path is absolute before changing directories.
427 filepath = os.path.abspath(filename)
428 saved_dir = os.getcwd()
429 os.chdir(output_dir)
430 command = unzip_cmd + [filepath]
431 result = bisect_utils.RunProcess(command)
432 os.chdir(saved_dir)
433 if result:
434 raise IOError('unzip failed: %s => %s' % (str(command), result))
435 else:
436 assert bisect_utils.IsWindowsHost() or bisect_utils.IsMacHost()
437 zf = zipfile.ZipFile(filename)
438 for name in zf.namelist():
439 if verbose:
440 print 'Extracting %s' % name
441 zf.extract(name, output_dir)
442 if bisect_utils.IsMacHost():
443 # Restore permission bits.
444 os.chmod(os.path.join(output_dir, name),
445 zf.getinfo(name).external_attr >> 16L)
446
447
448 def WriteStringToFile(text, file_name):
449 """Writes text to a file, raising an RuntimeError on failure."""
450 try:
451 with open(file_name, 'wb') as f:
452 f.write(text)
453 except IOError:
454 raise RuntimeError('Error writing to file [%s]' % file_name )
455
456
457 def ReadStringFromFile(file_name):
458 """Writes text to a file, raising an RuntimeError on failure."""
459 try:
460 with open(file_name) as f:
461 return f.read()
462 except IOError:
463 raise RuntimeError('Error reading file [%s]' % file_name )
464
465
466 def ChangeBackslashToSlashInPatch(diff_text):
467 """Formats file paths in the given patch text to Unix-style paths."""
468 if not diff_text:
469 return None
470 diff_lines = diff_text.split('\n')
471 for i in range(len(diff_lines)):
472 line = diff_lines[i]
473 if line.startswith('--- ') or line.startswith('+++ '):
474 diff_lines[i] = line.replace('\\', '/')
475 return '\n'.join(diff_lines)
476
477
478 def _ParseRevisionsFromDEPSFileManually(deps_file_contents):
479 """Parses the vars section of the DEPS file using regular expressions.
480
481 Args:
482 deps_file_contents: The DEPS file contents as a string.
483
484 Returns:
485 A dictionary in the format {depot: revision} if successful, otherwise None.
486 """
487 # We'll parse the "vars" section of the DEPS file.
488 rxp = re.compile('vars = {(?P<vars_body>[^}]+)', re.MULTILINE)
489 re_results = rxp.search(deps_file_contents)
490
491 if not re_results:
492 return None
493
494 # We should be left with a series of entries in the vars component of
495 # the DEPS file with the following format:
496 # 'depot_name': 'revision',
497 vars_body = re_results.group('vars_body')
498 rxp = re.compile("'(?P<depot_body>[\w_-]+)':[\s]+'(?P<rev_body>[\w@]+)'",
499 re.MULTILINE)
500 re_results = rxp.findall(vars_body)
501
502 return dict(re_results)
503
504
505 def _WaitUntilBuildIsReady(
506 fetch_build, bot_name, builder_host, builder_port, build_request_id,
507 max_timeout):
508 """Waits until build is produced by bisect builder on try server.
509
510 Args:
511 fetch_build: Function to check and download build from cloud storage.
512 bot_name: Builder bot name on try server.
513 builder_host Try server host name.
514 builder_port: Try server port.
515 build_request_id: A unique ID of the build request posted to try server.
516 max_timeout: Maximum time to wait for the build.
517
518 Returns:
519 Downloaded archive file path if exists, otherwise None.
520 """
521 # Build number on the try server.
522 build_num = None
523 # Interval to check build on cloud storage.
524 poll_interval = 60
525 # Interval to check build status on try server in seconds.
526 status_check_interval = 600
527 last_status_check = time.time()
528 start_time = time.time()
529 while True:
530 # Checks for build on gs://chrome-perf and download if exists.
531 res = fetch_build()
532 if res:
533 return (res, 'Build successfully found')
534 elapsed_status_check = time.time() - last_status_check
535 # To avoid overloading try server with status check requests, we check
536 # build status for every 10 minutes.
537 if elapsed_status_check > status_check_interval:
538 last_status_check = time.time()
539 if not build_num:
540 # Get the build number on try server for the current build.
541 build_num = request_build.GetBuildNumFromBuilder(
542 build_request_id, bot_name, builder_host, builder_port)
543 # Check the status of build using the build number.
544 # Note: Build is treated as PENDING if build number is not found
545 # on the the try server.
546 build_status, status_link = request_build.GetBuildStatus(
547 build_num, bot_name, builder_host, builder_port)
548 if build_status == request_build.FAILED:
549 return (None, 'Failed to produce build, log: %s' % status_link)
550 elapsed_time = time.time() - start_time
551 if elapsed_time > max_timeout:
552 return (None, 'Timed out: %ss without build' % max_timeout)
553
554 print 'Time elapsed: %ss without build.' % elapsed_time
555 time.sleep(poll_interval)
556 # For some reason, mac bisect bots were not flushing stdout periodically.
557 # As a result buildbot command is timed-out. Flush stdout on all platforms
558 # while waiting for build.
559 sys.stdout.flush()
560
561
562 def _UpdateV8Branch(deps_content):
563 """Updates V8 branch in DEPS file to process v8_bleeding_edge.
564
565 Check for "v8_branch" in DEPS file if exists update its value
566 with v8_bleeding_edge branch. Note: "v8_branch" is added to DEPS
567 variable from DEPS revision 254916, therefore check for "src/v8":
568 <v8 source path> in DEPS in order to support prior DEPS revisions
569 and update it.
570
571 Args:
572 deps_content: DEPS file contents to be modified.
573
574 Returns:
575 Modified DEPS file contents as a string.
576 """
577 new_branch = r'branches/bleeding_edge'
578 v8_branch_pattern = re.compile(r'(?<="v8_branch": ")(.*)(?=")')
579 if re.search(v8_branch_pattern, deps_content):
580 deps_content = re.sub(v8_branch_pattern, new_branch, deps_content)
581 else:
582 # Replaces the branch assigned to "src/v8" key in DEPS file.
583 # Format of "src/v8" in DEPS:
584 # "src/v8":
585 # (Var("googlecode_url") % "v8") + "/trunk@" + Var("v8_revision"),
586 # So, "/trunk@" is replace with "/branches/bleeding_edge@"
587 v8_src_pattern = re.compile(
588 r'(?<="v8"\) \+ "/)(.*)(?=@" \+ Var\("v8_revision"\))', re.MULTILINE)
589 if re.search(v8_src_pattern, deps_content):
590 deps_content = re.sub(v8_src_pattern, new_branch, deps_content)
591 return deps_content
592
593
594 def _UpdateDEPSForAngle(revision, depot, deps_file):
595 """Updates DEPS file with new revision for Angle repository.
596
597 This is a hack for Angle depot case because, in DEPS file "vars" dictionary
598 variable contains "angle_revision" key that holds git hash instead of
599 SVN revision.
600
601 And sometimes "angle_revision" key is not specified in "vars" variable,
602 in such cases check "deps" dictionary variable that matches
603 angle.git@[a-fA-F0-9]{40}$ and replace git hash.
604 """
605 deps_var = DEPOT_DEPS_NAME[depot]['deps_var']
606 try:
607 deps_contents = ReadStringFromFile(deps_file)
608 # Check whether the depot and revision pattern in DEPS file vars variable
609 # e.g. "angle_revision": "fa63e947cb3eccf463648d21a05d5002c9b8adfa".
610 angle_rev_pattern = re.compile(r'(?<="%s": ")([a-fA-F0-9]{40})(?=")' %
611 deps_var, re.MULTILINE)
612 match = re.search(angle_rev_pattern % deps_var, deps_contents)
613 if match:
614 # Update the revision information for the given depot
615 new_data = re.sub(angle_rev_pattern, revision, deps_contents)
616 else:
617 # Check whether the depot and revision pattern in DEPS file deps
618 # variable. e.g.,
619 # "src/third_party/angle": Var("chromium_git") +
620 # "/angle/angle.git@fa63e947cb3eccf463648d21a05d5002c9b8adfa",.
621 angle_rev_pattern = re.compile(
622 r'(?<=angle\.git@)([a-fA-F0-9]{40})(?=")', re.MULTILINE)
623 match = re.search(angle_rev_pattern, deps_contents)
624 if not match:
625 print 'Could not find angle revision information in DEPS file.'
626 return False
627 new_data = re.sub(angle_rev_pattern, revision, deps_contents)
628 # Write changes to DEPS file
629 WriteStringToFile(new_data, deps_file)
630 return True
631 except IOError, e:
632 print 'Something went wrong while updating DEPS file, %s' % e
633 return False
634
635
636 def _TryParseHistogramValuesFromOutput(metric, text):
637 """Attempts to parse a metric in the format HISTOGRAM <graph: <trace>.
638
639 Args:
640 metric: The metric as a list of [<trace>, <value>] strings.
641 text: The text to parse the metric values from.
642
643 Returns:
644 A list of floating point numbers found, [] if none were found.
645 """
646 metric_formatted = 'HISTOGRAM %s: %s= ' % (metric[0], metric[1])
647
648 text_lines = text.split('\n')
649 values_list = []
650
651 for current_line in text_lines:
652 if metric_formatted in current_line:
653 current_line = current_line[len(metric_formatted):]
654
655 try:
656 histogram_values = eval(current_line)
657
658 for b in histogram_values['buckets']:
659 average_for_bucket = float(b['high'] + b['low']) * 0.5
660 # Extends the list with N-elements with the average for that bucket.
661 values_list.extend([average_for_bucket] * b['count'])
662 except Exception:
663 pass
664
665 return values_list
666
667
668 def _TryParseResultValuesFromOutput(metric, text):
669 """Attempts to parse a metric in the format RESULT <graph>: <trace>= ...
670
671 Args:
672 metric: The metric as a list of [<trace>, <value>] string pairs.
673 text: The text to parse the metric values from.
674
675 Returns:
676 A list of floating point numbers found.
677 """
678 # Format is: RESULT <graph>: <trace>= <value> <units>
679 metric_re = re.escape('RESULT %s: %s=' % (metric[0], metric[1]))
680
681 # The log will be parsed looking for format:
682 # <*>RESULT <graph_name>: <trace_name>= <value>
683 single_result_re = re.compile(
684 metric_re + '\s*(?P<VALUE>[-]?\d*(\.\d*)?)')
685
686 # The log will be parsed looking for format:
687 # <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...]
688 multi_results_re = re.compile(
689 metric_re + '\s*\[\s*(?P<VALUES>[-]?[\d\., ]+)\s*\]')
690
691 # The log will be parsed looking for format:
692 # <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>}
693 mean_stddev_re = re.compile(
694 metric_re +
695 '\s*\{\s*(?P<MEAN>[-]?\d*(\.\d*)?),\s*(?P<STDDEV>\d+(\.\d*)?)\s*\}')
696
697 text_lines = text.split('\n')
698 values_list = []
699 for current_line in text_lines:
700 # Parse the output from the performance test for the metric we're
701 # interested in.
702 single_result_match = single_result_re.search(current_line)
703 multi_results_match = multi_results_re.search(current_line)
704 mean_stddev_match = mean_stddev_re.search(current_line)
705 if (not single_result_match is None and
706 single_result_match.group('VALUE')):
707 values_list += [single_result_match.group('VALUE')]
708 elif (not multi_results_match is None and
709 multi_results_match.group('VALUES')):
710 metric_values = multi_results_match.group('VALUES')
711 values_list += metric_values.split(',')
712 elif (not mean_stddev_match is None and
713 mean_stddev_match.group('MEAN')):
714 values_list += [mean_stddev_match.group('MEAN')]
715
716 values_list = [float(v) for v in values_list
717 if bisect_utils.IsStringFloat(v)]
718
719 # If the metric is times/t, we need to sum the timings in order to get
720 # similar regression results as the try-bots.
721 metrics_to_sum = [
722 ['times', 't'],
723 ['times', 'page_load_time'],
724 ['cold_times', 'page_load_time'],
725 ['warm_times', 'page_load_time'],
726 ]
727
728 if metric in metrics_to_sum:
729 if values_list:
730 values_list = [reduce(lambda x, y: float(x) + float(y), values_list)]
731
732 return values_list
733
734
735 def _ParseMetricValuesFromOutput(metric, text):
736 """Parses output from performance_ui_tests and retrieves the results for
737 a given metric.
738
739 Args:
740 metric: The metric as a list of [<trace>, <value>] strings.
741 text: The text to parse the metric values from.
742
743 Returns:
744 A list of floating point numbers found.
745 """
746 metric_values = _TryParseResultValuesFromOutput(metric, text)
747
748 if not metric_values:
749 metric_values = _TryParseHistogramValuesFromOutput(metric, text)
750
751 return metric_values
752
753
754 def _GenerateProfileIfNecessary(command_args):
755 """Checks the command line of the performance test for dependencies on
756 profile generation, and runs tools/perf/generate_profile as necessary.
757
758 Args:
759 command_args: Command line being passed to performance test, as a list.
760
761 Returns:
762 False if profile generation was necessary and failed, otherwise True.
763 """
764 if '--profile-dir' in ' '.join(command_args):
765 # If we were using python 2.7+, we could just use the argparse
766 # module's parse_known_args to grab --profile-dir. Since some of the
767 # bots still run 2.6, have to grab the arguments manually.
768 arg_dict = {}
769 args_to_parse = ['--profile-dir', '--browser']
770
771 for arg_to_parse in args_to_parse:
772 for i, current_arg in enumerate(command_args):
773 if arg_to_parse in current_arg:
774 current_arg_split = current_arg.split('=')
775
776 # Check 2 cases, --arg=<val> and --arg <val>
777 if len(current_arg_split) == 2:
778 arg_dict[arg_to_parse] = current_arg_split[1]
779 elif i + 1 < len(command_args):
780 arg_dict[arg_to_parse] = command_args[i+1]
781
782 path_to_generate = os.path.join('tools', 'perf', 'generate_profile')
783
784 if arg_dict.has_key('--profile-dir') and arg_dict.has_key('--browser'):
785 profile_path, profile_type = os.path.split(arg_dict['--profile-dir'])
786 return not bisect_utils.RunProcess(['python', path_to_generate,
787 '--profile-type-to-generate', profile_type,
788 '--browser', arg_dict['--browser'], '--output-dir', profile_path])
789 return False
790 return True
791
792
793 def _AddRevisionsIntoRevisionData(revisions, depot, sort, revision_data):
794 """Adds new revisions to the revision_data dictionary and initializes them.
795
796 Args:
797 revisions: List of revisions to add.
798 depot: Depot that's currently in use (src, webkit, etc...)
799 sort: Sorting key for displaying revisions.
800 revision_data: A dictionary to add the new revisions into.
801 Existing revisions will have their sort keys adjusted.
802 """
803 num_depot_revisions = len(revisions)
804
805 for _, v in revision_data.iteritems():
806 if v['sort'] > sort:
807 v['sort'] += num_depot_revisions
808
809 for i in xrange(num_depot_revisions):
810 r = revisions[i]
811 revision_data[r] = {
812 'revision' : r,
813 'depot' : depot,
814 'value' : None,
815 'perf_time' : 0,
816 'build_time' : 0,
817 'passed' : '?',
818 'sort' : i + sort + 1,
819 }
820
821
822 def _PrintThankYou():
823 print RESULTS_THANKYOU
824
825
826 def _PrintTableRow(column_widths, row_data):
827 """Prints out a row in a formatted table that has columns aligned.
828
829 Args:
830 column_widths: A list of column width numbers.
831 row_data: A list of items for each column in this row.
832 """
833 assert len(column_widths) == len(row_data)
834 text = ''
835 for i in xrange(len(column_widths)):
836 current_row_data = row_data[i].center(column_widths[i], ' ')
837 text += ('%%%ds' % column_widths[i]) % current_row_data
838 print text
839
840
841 def _PrintStepTime(revision_data_sorted):
842 """Prints information about how long various steps took.
843
844 Args:
845 revision_data_sorted: The sorted list of revision data dictionaries."""
846 step_perf_time_avg = 0.0
847 step_build_time_avg = 0.0
848 step_count = 0.0
849 for _, current_data in revision_data_sorted:
850 if current_data['value']:
851 step_perf_time_avg += current_data['perf_time']
852 step_build_time_avg += current_data['build_time']
853 step_count += 1
854 if step_count:
855 step_perf_time_avg = step_perf_time_avg / step_count
856 step_build_time_avg = step_build_time_avg / step_count
857 print
858 print 'Average build time : %s' % datetime.timedelta(
859 seconds=int(step_build_time_avg))
860 print 'Average test time : %s' % datetime.timedelta(
861 seconds=int(step_perf_time_avg))
862
863
864 def _FindOtherRegressions(revision_data_sorted, bad_greater_than_good):
865 """Compiles a list of other possible regressions from the revision data.
866
867 Args:
868 revision_data_sorted: Sorted list of (revision, revision data) pairs.
869 bad_greater_than_good: Whether the result value at the "bad" revision is
870 numerically greater than the result value at the "good" revision.
871
872 Returns:
873 A list of [current_rev, previous_rev, confidence] for other places where
874 there may have been a regression.
875 """
876 other_regressions = []
877 previous_values = []
878 previous_id = None
879 for current_id, current_data in revision_data_sorted:
880 current_values = current_data['value']
881 if current_values:
882 current_values = current_values['values']
883 if previous_values:
884 confidence = ConfidenceScore(previous_values, [current_values])
885 mean_of_prev_runs = math_utils.Mean(sum(previous_values, []))
886 mean_of_current_runs = math_utils.Mean(current_values)
887
888 # Check that the potential regression is in the same direction as
889 # the overall regression. If the mean of the previous runs < the
890 # mean of the current runs, this local regression is in same
891 # direction.
892 prev_less_than_current = mean_of_prev_runs < mean_of_current_runs
893 is_same_direction = (prev_less_than_current if
894 bad_greater_than_good else not prev_less_than_current)
895
896 # Only report potential regressions with high confidence.
897 if is_same_direction and confidence > 50:
898 other_regressions.append([current_id, previous_id, confidence])
899 previous_values.append(current_values)
900 previous_id = current_id
901 return other_regressions
902
903
904 class BisectPerformanceMetrics(object):
905 """This class contains functionality to perform a bisection of a range of
906 revisions to narrow down where performance regressions may have occurred.
907
908 The main entry-point is the Run method.
909 """
910
911 def __init__(self, source_control, opts):
912 super(BisectPerformanceMetrics, self).__init__()
913
914 self.opts = opts
915 self.source_control = source_control
916 self.src_cwd = os.getcwd()
917 self.cros_cwd = os.path.join(os.getcwd(), '..', 'cros')
918 self.depot_cwd = {}
919 self.cleanup_commands = []
920 self.warnings = []
921 self.builder = builder.Builder.FromOpts(opts)
922
923 for d in DEPOT_NAMES:
924 # The working directory of each depot is just the path to the depot, but
925 # since we're already in 'src', we can skip that part.
926
927 self.depot_cwd[d] = os.path.join(
928 self.src_cwd, DEPOT_DEPS_NAME[d]['src'][4:])
929
930 def PerformCleanup(self):
931 """Performs cleanup when script is finished."""
932 os.chdir(self.src_cwd)
933 for c in self.cleanup_commands:
934 if c[0] == 'mv':
935 shutil.move(c[1], c[2])
936 else:
937 assert False, 'Invalid cleanup command.'
938
939 def GetRevisionList(self, depot, bad_revision, good_revision):
940 """Retrieves a list of all the commits between the bad revision and
941 last known good revision."""
942
943 revision_work_list = []
944
945 if depot == 'cros':
946 revision_range_start = good_revision
947 revision_range_end = bad_revision
948
949 cwd = os.getcwd()
950 self.ChangeToDepotWorkingDirectory('cros')
951
952 # Print the commit timestamps for every commit in the revision time
953 # range. We'll sort them and bisect by that. There is a remote chance that
954 # 2 (or more) commits will share the exact same timestamp, but it's
955 # probably safe to ignore that case.
956 cmd = ['repo', 'forall', '-c',
957 'git log --format=%%ct --before=%d --after=%d' % (
958 revision_range_end, revision_range_start)]
959 output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd)
960
961 assert not return_code, ('An error occurred while running '
962 '"%s"' % ' '.join(cmd))
963
964 os.chdir(cwd)
965
966 revision_work_list = list(set(
967 [int(o) for o in output.split('\n') if bisect_utils.IsStringInt(o)]))
968 revision_work_list = sorted(revision_work_list, reverse=True)
969 else:
970 cwd = self._GetDepotDirectory(depot)
971 revision_work_list = self.source_control.GetRevisionList(bad_revision,
972 good_revision, cwd=cwd)
973
974 return revision_work_list
975
976 def _GetV8BleedingEdgeFromV8TrunkIfMappable(self, revision):
977 commit_position = self.source_control.GetCommitPosition(revision)
978
979 if bisect_utils.IsStringInt(commit_position):
980 # V8 is tricky to bisect, in that there are only a few instances when
981 # we can dive into bleeding_edge and get back a meaningful result.
982 # Try to detect a V8 "business as usual" case, which is when:
983 # 1. trunk revision N has description "Version X.Y.Z"
984 # 2. bleeding_edge revision (N-1) has description "Prepare push to
985 # trunk. Now working on X.Y.(Z+1)."
986 #
987 # As of 01/24/2014, V8 trunk descriptions are formatted:
988 # "Version 3.X.Y (based on bleeding_edge revision rZ)"
989 # So we can just try parsing that out first and fall back to the old way.
990 v8_dir = self._GetDepotDirectory('v8')
991 v8_bleeding_edge_dir = self._GetDepotDirectory('v8_bleeding_edge')
992
993 revision_info = self.source_control.QueryRevisionInfo(revision,
994 cwd=v8_dir)
995
996 version_re = re.compile("Version (?P<values>[0-9,.]+)")
997
998 regex_results = version_re.search(revision_info['subject'])
999
1000 if regex_results:
1001 git_revision = None
1002
1003 # Look for "based on bleeding_edge" and parse out revision
1004 if 'based on bleeding_edge' in revision_info['subject']:
1005 try:
1006 bleeding_edge_revision = revision_info['subject'].split(
1007 'bleeding_edge revision r')[1]
1008 bleeding_edge_revision = int(bleeding_edge_revision.split(')')[0])
1009 git_revision = self.source_control.ResolveToRevision(
1010 bleeding_edge_revision, 'v8_bleeding_edge', DEPOT_DEPS_NAME, 1,
1011 cwd=v8_bleeding_edge_dir)
1012 return git_revision
1013 except (IndexError, ValueError):
1014 pass
1015
1016 if not git_revision:
1017 # Wasn't successful, try the old way of looking for "Prepare push to"
1018 git_revision = self.source_control.ResolveToRevision(
1019 int(commit_position) - 1, 'v8_bleeding_edge', DEPOT_DEPS_NAME, -1,
1020 cwd=v8_bleeding_edge_dir)
1021
1022 if git_revision:
1023 revision_info = self.source_control.QueryRevisionInfo(git_revision,
1024 cwd=v8_bleeding_edge_dir)
1025
1026 if 'Prepare push to trunk' in revision_info['subject']:
1027 return git_revision
1028 return None
1029
1030 def _GetNearestV8BleedingEdgeFromTrunk(self, revision, search_forward=True):
1031 cwd = self._GetDepotDirectory('v8')
1032 cmd = ['log', '--format=%ct', '-1', revision]
1033 output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
1034 commit_time = int(output)
1035 commits = []
1036
1037 if search_forward:
1038 cmd = ['log', '--format=%H', '-10', '--after=%d' % commit_time,
1039 'origin/master']
1040 output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
1041 output = output.split()
1042 commits = output
1043 commits = reversed(commits)
1044 else:
1045 cmd = ['log', '--format=%H', '-10', '--before=%d' % commit_time,
1046 'origin/master']
1047 output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
1048 output = output.split()
1049 commits = output
1050
1051 bleeding_edge_revision = None
1052
1053 for c in commits:
1054 bleeding_edge_revision = self._GetV8BleedingEdgeFromV8TrunkIfMappable(c)
1055 if bleeding_edge_revision:
1056 break
1057
1058 return bleeding_edge_revision
1059
1060 def _ParseRevisionsFromDEPSFile(self, depot):
1061 """Parses the local DEPS file to determine blink/skia/v8 revisions which may
1062 be needed if the bisect recurses into those depots later.
1063
1064 Args:
1065 depot: Name of depot being bisected.
1066
1067 Returns:
1068 A dict in the format {depot:revision} if successful, otherwise None.
1069 """
1070 try:
1071 deps_data = {
1072 'Var': lambda _: deps_data["vars"][_],
1073 'From': lambda *args: None,
1074 }
1075
1076 deps_file = bisect_utils.FILE_DEPS_GIT
1077 if not os.path.exists(deps_file):
1078 deps_file = bisect_utils.FILE_DEPS
1079 execfile(deps_file, {}, deps_data)
1080 deps_data = deps_data['deps']
1081
1082 rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)")
1083 results = {}
1084 for depot_name, depot_data in DEPOT_DEPS_NAME.iteritems():
1085 if (depot_data.get('platform') and
1086 depot_data.get('platform') != os.name):
1087 continue
1088
1089 if (depot_data.get('recurse') and depot in depot_data.get('from')):
1090 depot_data_src = depot_data.get('src') or depot_data.get('src_old')
1091 src_dir = deps_data.get(depot_data_src)
1092 if src_dir:
1093 self.depot_cwd[depot_name] = os.path.join(self.src_cwd,
1094 depot_data_src[4:])
1095 re_results = rxp.search(src_dir)
1096 if re_results:
1097 results[depot_name] = re_results.group('revision')
1098 else:
1099 warning_text = ('Could not parse revision for %s while bisecting '
1100 '%s' % (depot_name, depot))
1101 if not warning_text in self.warnings:
1102 self.warnings.append(warning_text)
1103 else:
1104 results[depot_name] = None
1105 return results
1106 except ImportError:
1107 deps_file_contents = ReadStringFromFile(deps_file)
1108 parse_results = _ParseRevisionsFromDEPSFileManually(deps_file_contents)
1109 results = {}
1110 for depot_name, depot_revision in parse_results.iteritems():
1111 depot_revision = depot_revision.strip('@')
1112 print depot_name, depot_revision
1113 for current_name, current_data in DEPOT_DEPS_NAME.iteritems():
1114 if (current_data.has_key('deps_var') and
1115 current_data['deps_var'] == depot_name):
1116 src_name = current_name
1117 results[src_name] = depot_revision
1118 break
1119 return results
1120
1121 def _Get3rdPartyRevisions(self, depot):
1122 """Parses the DEPS file to determine WebKit/v8/etc... versions.
1123
1124 Args:
1125 depot: A depot name. Should be in the DEPOT_NAMES list.
1126
1127 Returns:
1128 A dict in the format {depot: revision} if successful, otherwise None.
1129 """
1130 cwd = os.getcwd()
1131 self.ChangeToDepotWorkingDirectory(depot)
1132
1133 results = {}
1134
1135 if depot == 'chromium' or depot == 'android-chrome':
1136 results = self._ParseRevisionsFromDEPSFile(depot)
1137 os.chdir(cwd)
1138
1139 if depot == 'cros':
1140 cmd = [
1141 bisect_utils.CROS_SDK_PATH,
1142 '--',
1143 'portageq-%s' % self.opts.cros_board,
1144 'best_visible',
1145 '/build/%s' % self.opts.cros_board,
1146 'ebuild',
1147 CROS_CHROMEOS_PATTERN
1148 ]
1149 output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd)
1150
1151 assert not return_code, ('An error occurred while running '
1152 '"%s"' % ' '.join(cmd))
1153
1154 if len(output) > CROS_CHROMEOS_PATTERN:
1155 output = output[len(CROS_CHROMEOS_PATTERN):]
1156
1157 if len(output) > 1:
1158 output = output.split('_')[0]
1159
1160 if len(output) > 3:
1161 contents = output.split('.')
1162
1163 version = contents[2]
1164
1165 if contents[3] != '0':
1166 warningText = ('Chrome version: %s.%s but using %s.0 to bisect.' %
1167 (version, contents[3], version))
1168 if not warningText in self.warnings:
1169 self.warnings.append(warningText)
1170
1171 cwd = os.getcwd()
1172 self.ChangeToDepotWorkingDirectory('chromium')
1173 cmd = ['log', '-1', '--format=%H',
1174 '--author=chrome-release@google.com',
1175 '--grep=to %s' % version, 'origin/master']
1176 return_code = bisect_utils.CheckRunGit(cmd)
1177 os.chdir(cwd)
1178
1179 results['chromium'] = output.strip()
1180
1181 if depot == 'v8':
1182 # We can't try to map the trunk revision to bleeding edge yet, because
1183 # we don't know which direction to try to search in. Have to wait until
1184 # the bisect has narrowed the results down to 2 v8 rolls.
1185 results['v8_bleeding_edge'] = None
1186
1187 return results
1188
1189 def BackupOrRestoreOutputDirectory(self, restore=False, build_type='Release'):
1190 """Backs up or restores build output directory based on restore argument.
1191
1192 Args:
1193 restore: Indicates whether to restore or backup. Default is False(Backup)
1194 build_type: Target build type ('Release', 'Debug', 'Release_x64' etc.)
1195
1196 Returns:
1197 Path to backup or restored location as string. otherwise None if it fails.
1198 """
1199 build_dir = os.path.abspath(
1200 builder.GetBuildOutputDirectory(self.opts, self.src_cwd))
1201 source_dir = os.path.join(build_dir, build_type)
1202 destination_dir = os.path.join(build_dir, '%s.bak' % build_type)
1203 if restore:
1204 source_dir, destination_dir = destination_dir, source_dir
1205 if os.path.exists(source_dir):
1206 RmTreeAndMkDir(destination_dir, skip_makedir=True)
1207 shutil.move(source_dir, destination_dir)
1208 return destination_dir
1209 return None
1210
1211 def GetBuildArchiveForRevision(self, revision, gs_bucket, target_arch,
1212 patch_sha, out_dir):
1213 """Checks and downloads build archive for a given revision.
1214
1215 Checks for build archive with Git hash or SVN revision. If either of the
1216 file exists, then downloads the archive file.
1217
1218 Args:
1219 revision: A Git hash revision.
1220 gs_bucket: Cloud storage bucket name
1221 target_arch: 32 or 64 bit build target
1222 patch: A DEPS patch (used while bisecting 3rd party repositories).
1223 out_dir: Build output directory where downloaded file is stored.
1224
1225 Returns:
1226 Downloaded archive file path if exists, otherwise None.
1227 """
1228 # Source archive file path on cloud storage using Git revision.
1229 source_file = GetRemoteBuildPath(
1230 revision, self.opts.target_platform, target_arch, patch_sha)
1231 downloaded_archive = FetchFromCloudStorage(gs_bucket, source_file, out_dir)
1232 if not downloaded_archive:
1233 # Get commit position for the given SHA.
1234 commit_position = self.source_control.GetCommitPosition(revision)
1235 if commit_position:
1236 # Source archive file path on cloud storage using SVN revision.
1237 source_file = GetRemoteBuildPath(
1238 commit_position, self.opts.target_platform, target_arch, patch_sha)
1239 return FetchFromCloudStorage(gs_bucket, source_file, out_dir)
1240 return downloaded_archive
1241
1242 def DownloadCurrentBuild(self, revision, build_type='Release', patch=None):
1243 """Downloads the build archive for the given revision.
1244
1245 Args:
1246 revision: The Git revision to download or build.
1247 build_type: Target build type ('Release', 'Debug', 'Release_x64' etc.)
1248 patch: A DEPS patch (used while bisecting 3rd party repositories).
1249
1250 Returns:
1251 True if download succeeds, otherwise False.
1252 """
1253 patch_sha = None
1254 if patch:
1255 # Get the SHA of the DEPS changes patch.
1256 patch_sha = GetSHA1HexDigest(patch)
1257
1258 # Update the DEPS changes patch with a patch to create a new file named
1259 # 'DEPS.sha' and add patch_sha evaluated above to it.
1260 patch = '%s\n%s' % (patch, DEPS_SHA_PATCH % {'deps_sha': patch_sha})
1261
1262 # Get Build output directory
1263 abs_build_dir = os.path.abspath(
1264 builder.GetBuildOutputDirectory(self.opts, self.src_cwd))
1265
1266 fetch_build_func = lambda: self.GetBuildArchiveForRevision(
1267 revision, self.opts.gs_bucket, self.opts.target_arch,
1268 patch_sha, abs_build_dir)
1269
1270 # Downloaded archive file path, downloads build archive for given revision.
1271 downloaded_file = fetch_build_func()
1272
1273 # When build archive doesn't exists, post a build request to tryserver
1274 # and wait for the build to be produced.
1275 if not downloaded_file:
1276 downloaded_file = self.PostBuildRequestAndWait(
1277 revision, fetch_build=fetch_build_func, patch=patch)
1278 if not downloaded_file:
1279 return False
1280
1281 # Generic name for the archive, created when archive file is extracted.
1282 output_dir = os.path.join(
1283 abs_build_dir, GetZipFileName(target_arch=self.opts.target_arch))
1284 # Unzip build archive directory.
1285 try:
1286 RmTreeAndMkDir(output_dir, skip_makedir=True)
1287 self.BackupOrRestoreOutputDirectory(restore=False)
1288 # Build output directory based on target(e.g. out/Release, out/Debug).
1289 target_build_output_dir = os.path.join(abs_build_dir, build_type)
1290 ExtractZip(downloaded_file, abs_build_dir)
1291 if not os.path.exists(output_dir):
1292 # Due to recipe changes, the builds extract folder contains
1293 # out/Release instead of full-build-<platform>/Release.
1294 if os.path.exists(os.path.join(abs_build_dir, 'out', build_type)):
1295 output_dir = os.path.join(abs_build_dir, 'out', build_type)
1296 else:
1297 raise IOError('Missing extracted folder %s ' % output_dir)
1298
1299 print 'Moving build from %s to %s' % (
1300 output_dir, target_build_output_dir)
1301 shutil.move(output_dir, target_build_output_dir)
1302 return True
1303 except Exception as e:
1304 print 'Something went wrong while extracting archive file: %s' % e
1305 self.BackupOrRestoreOutputDirectory(restore=True)
1306 # Cleanup any leftovers from unzipping.
1307 if os.path.exists(output_dir):
1308 RmTreeAndMkDir(output_dir, skip_makedir=True)
1309 finally:
1310 # Delete downloaded archive
1311 if os.path.exists(downloaded_file):
1312 os.remove(downloaded_file)
1313 return False
1314
1315 def PostBuildRequestAndWait(self, git_revision, fetch_build, patch=None):
1316 """POSTs the build request job to the try server instance.
1317
1318 A try job build request is posted to tryserver.chromium.perf master,
1319 and waits for the binaries to be produced and archived on cloud storage.
1320 Once the build is ready and stored onto cloud, build archive is downloaded
1321 into the output folder.
1322
1323 Args:
1324 git_revision: A Git hash revision.
1325 fetch_build: Function to check and download build from cloud storage.
1326 patch: A DEPS patch (used while bisecting 3rd party repositories).
1327
1328 Returns:
1329 Downloaded archive file path when requested build exists and download is
1330 successful, otherwise None.
1331 """
1332 def GetBuilderNameAndBuildTime(target_platform, target_arch='ia32'):
1333 """Gets builder bot name and build time in seconds based on platform."""
1334 # Bot names should match the one listed in tryserver.chromium's
1335 # master.cfg which produces builds for bisect.
1336 if bisect_utils.IsWindowsHost():
1337 if bisect_utils.Is64BitWindows() and target_arch == 'x64':
1338 return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME)
1339 return ('win_perf_bisect_builder', MAX_WIN_BUILD_TIME)
1340 if bisect_utils.IsLinuxHost():
1341 if target_platform == 'android':
1342 return ('android_perf_bisect_builder', MAX_LINUX_BUILD_TIME)
1343 return ('linux_perf_bisect_builder', MAX_LINUX_BUILD_TIME)
1344 if bisect_utils.IsMacHost():
1345 return ('mac_perf_bisect_builder', MAX_MAC_BUILD_TIME)
1346 raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
1347 if not fetch_build:
1348 return False
1349
1350 bot_name, build_timeout = GetBuilderNameAndBuildTime(
1351 self.opts.target_platform, self.opts.target_arch)
1352 builder_host = self.opts.builder_host
1353 builder_port = self.opts.builder_port
1354 # Create a unique ID for each build request posted to try server builders.
1355 # This ID is added to "Reason" property of the build.
1356 build_request_id = GetSHA1HexDigest(
1357 '%s-%s-%s' % (git_revision, patch, time.time()))
1358
1359 # Creates a try job description.
1360 # Always use Git hash to post build request since Commit positions are
1361 # not supported by builders to build.
1362 job_args = {
1363 'revision': 'src@%s' % git_revision,
1364 'bot': bot_name,
1365 'name': build_request_id,
1366 }
1367 # Update patch information if supplied.
1368 if patch:
1369 job_args['patch'] = patch
1370 # Posts job to build the revision on the server.
1371 if request_build.PostTryJob(builder_host, builder_port, job_args):
1372 target_file, error_msg = _WaitUntilBuildIsReady(
1373 fetch_build, bot_name, builder_host, builder_port, build_request_id,
1374 build_timeout)
1375 if not target_file:
1376 print '%s [revision: %s]' % (error_msg, git_revision)
1377 return None
1378 return target_file
1379 print 'Failed to post build request for revision: [%s]' % git_revision
1380 return None
1381
1382 def IsDownloadable(self, depot):
1383 """Checks if build can be downloaded based on target platform and depot."""
1384 if (self.opts.target_platform in ['chromium', 'android'] and
1385 self.opts.gs_bucket):
1386 return (depot == 'chromium' or
1387 'chromium' in DEPOT_DEPS_NAME[depot]['from'] or
1388 'v8' in DEPOT_DEPS_NAME[depot]['from'])
1389 return False
1390
1391 def UpdateDepsContents(self, deps_contents, depot, git_revision, deps_key):
1392 """Returns modified version of DEPS file contents.
1393
1394 Args:
1395 deps_contents: DEPS file content.
1396 depot: Current depot being bisected.
1397 git_revision: A git hash to be updated in DEPS.
1398 deps_key: Key in vars section of DEPS file to be searched.
1399
1400 Returns:
1401 Updated DEPS content as string if deps key is found, otherwise None.
1402 """
1403 # Check whether the depot and revision pattern in DEPS file vars
1404 # e.g. for webkit the format is "webkit_revision": "12345".
1405 deps_revision = re.compile(r'(?<="%s": ")([0-9]+)(?=")' % deps_key,
1406 re.MULTILINE)
1407 new_data = None
1408 if re.search(deps_revision, deps_contents):
1409 commit_position = self.source_control.GetCommitPosition(
1410 git_revision, self._GetDepotDirectory(depot))
1411 if not commit_position:
1412 print 'Could not determine commit position for %s' % git_revision
1413 return None
1414 # Update the revision information for the given depot
1415 new_data = re.sub(deps_revision, str(commit_position), deps_contents)
1416 else:
1417 # Check whether the depot and revision pattern in DEPS file vars
1418 # e.g. for webkit the format is "webkit_revision": "559a6d4ab7a84c539..".
1419 deps_revision = re.compile(
1420 r'(?<=["\']%s["\']: ["\'])([a-fA-F0-9]{40})(?=["\'])' % deps_key,
1421 re.MULTILINE)
1422 if re.search(deps_revision, deps_contents):
1423 new_data = re.sub(deps_revision, git_revision, deps_contents)
1424 if new_data:
1425 # For v8_bleeding_edge revisions change V8 branch in order
1426 # to fetch bleeding edge revision.
1427 if depot == 'v8_bleeding_edge':
1428 new_data = _UpdateV8Branch(new_data)
1429 if not new_data:
1430 return None
1431 return new_data
1432
1433 def UpdateDeps(self, revision, depot, deps_file):
1434 """Updates DEPS file with new revision of dependency repository.
1435
1436 This method search DEPS for a particular pattern in which depot revision
1437 is specified (e.g "webkit_revision": "123456"). If a match is found then
1438 it resolves the given git hash to SVN revision and replace it in DEPS file.
1439
1440 Args:
1441 revision: A git hash revision of the dependency repository.
1442 depot: Current depot being bisected.
1443 deps_file: Path to DEPS file.
1444
1445 Returns:
1446 True if DEPS file is modified successfully, otherwise False.
1447 """
1448 if not os.path.exists(deps_file):
1449 return False
1450
1451 deps_var = DEPOT_DEPS_NAME[depot]['deps_var']
1452 # Don't update DEPS file if deps_var is not set in DEPOT_DEPS_NAME.
1453 if not deps_var:
1454 print 'DEPS update not supported for Depot: %s', depot
1455 return False
1456
1457 # Hack for Angle repository. In the DEPS file, "vars" dictionary variable
1458 # contains "angle_revision" key that holds git hash instead of SVN revision.
1459 # And sometime "angle_revision" key is not specified in "vars" variable.
1460 # In such cases check, "deps" dictionary variable that matches
1461 # angle.git@[a-fA-F0-9]{40}$ and replace git hash.
1462 if depot == 'angle':
1463 return _UpdateDEPSForAngle(revision, depot, deps_file)
1464
1465 try:
1466 deps_contents = ReadStringFromFile(deps_file)
1467 updated_deps_content = self.UpdateDepsContents(
1468 deps_contents, depot, revision, deps_var)
1469 # Write changes to DEPS file
1470 if updated_deps_content:
1471 WriteStringToFile(updated_deps_content, deps_file)
1472 return True
1473 except IOError, e:
1474 print 'Something went wrong while updating DEPS file. [%s]' % e
1475 return False
1476
1477 def CreateDEPSPatch(self, depot, revision):
1478 """Modifies DEPS and returns diff as text.
1479
1480 Args:
1481 depot: Current depot being bisected.
1482 revision: A git hash revision of the dependency repository.
1483
1484 Returns:
1485 A tuple with git hash of chromium revision and DEPS patch text.
1486 """
1487 deps_file_path = os.path.join(self.src_cwd, bisect_utils.FILE_DEPS)
1488 if not os.path.exists(deps_file_path):
1489 raise RuntimeError('DEPS file does not exists.[%s]' % deps_file_path)
1490 # Get current chromium revision (git hash).
1491 cmd = ['rev-parse', 'HEAD']
1492 chromium_sha = bisect_utils.CheckRunGit(cmd).strip()
1493 if not chromium_sha:
1494 raise RuntimeError('Failed to determine Chromium revision for %s' %
1495 revision)
1496 if ('chromium' in DEPOT_DEPS_NAME[depot]['from'] or
1497 'v8' in DEPOT_DEPS_NAME[depot]['from']):
1498 # Checkout DEPS file for the current chromium revision.
1499 if self.source_control.CheckoutFileAtRevision(
1500 bisect_utils.FILE_DEPS, chromium_sha, cwd=self.src_cwd):
1501 if self.UpdateDeps(revision, depot, deps_file_path):
1502 diff_command = [
1503 'diff',
1504 '--src-prefix=src/',
1505 '--dst-prefix=src/',
1506 '--no-ext-diff',
1507 bisect_utils.FILE_DEPS,
1508 ]
1509 diff_text = bisect_utils.CheckRunGit(diff_command, cwd=self.src_cwd)
1510 return (chromium_sha, ChangeBackslashToSlashInPatch(diff_text))
1511 else:
1512 raise RuntimeError(
1513 'Failed to update DEPS file for chromium: [%s]' % chromium_sha)
1514 else:
1515 raise RuntimeError(
1516 'DEPS checkout Failed for chromium revision : [%s]' % chromium_sha)
1517 return (None, None)
1518
1519 def BuildCurrentRevision(self, depot, revision=None):
1520 """Builds chrome and performance_ui_tests on the current revision.
1521
1522 Returns:
1523 True if the build was successful.
1524 """
1525 if self.opts.debug_ignore_build:
1526 return True
1527
1528 build_success = False
1529 cwd = os.getcwd()
1530 os.chdir(self.src_cwd)
1531 # Fetch build archive for the given revision from the cloud storage when
1532 # the storage bucket is passed.
1533 if self.IsDownloadable(depot) and revision:
1534 deps_patch = None
1535 if depot != 'chromium':
1536 # Create a DEPS patch with new revision for dependency repository.
1537 revision, deps_patch = self.CreateDEPSPatch(depot, revision)
1538 if self.DownloadCurrentBuild(revision, patch=deps_patch):
1539 if deps_patch:
1540 # Reverts the changes to DEPS file.
1541 self.source_control.CheckoutFileAtRevision(
1542 bisect_utils.FILE_DEPS, revision, cwd=self.src_cwd)
1543 build_success = True
1544 else:
1545 # These codes are executed when bisect bots builds binaries locally.
1546 build_success = self.builder.Build(depot, self.opts)
1547 os.chdir(cwd)
1548 return build_success
1549
1550 def RunGClientHooks(self):
1551 """Runs gclient with runhooks command.
1552
1553 Returns:
1554 True if gclient reports no errors.
1555 """
1556 if self.opts.debug_ignore_build:
1557 return True
1558 return not bisect_utils.RunGClient(['runhooks'], cwd=self.src_cwd)
1559
1560 def _IsBisectModeUsingMetric(self):
1561 return self.opts.bisect_mode in [BISECT_MODE_MEAN, BISECT_MODE_STD_DEV]
1562
1563 def _IsBisectModeReturnCode(self):
1564 return self.opts.bisect_mode in [BISECT_MODE_RETURN_CODE]
1565
1566 def _IsBisectModeStandardDeviation(self):
1567 return self.opts.bisect_mode in [BISECT_MODE_STD_DEV]
1568
1569 def GetCompatibleCommand(self, command_to_run, revision, depot):
1570 # Prior to crrev.com/274857 *only* android-chromium-testshell
1571 # Then until crrev.com/276628 *both* (android-chromium-testshell and
1572 # android-chrome-shell) work. After that rev 276628 *only*
1573 # android-chrome-shell works. bisect-perf-regression.py script should
1574 # handle these cases and set appropriate browser type based on revision.
1575 if self.opts.target_platform in ['android']:
1576 # When its a third_party depot, get the chromium revision.
1577 if depot != 'chromium':
1578 revision = bisect_utils.CheckRunGit(
1579 ['rev-parse', 'HEAD'], cwd=self.src_cwd).strip()
1580 commit_position = self.source_control.GetCommitPosition(revision,
1581 cwd=self.src_cwd)
1582 if not commit_position:
1583 return command_to_run
1584 cmd_re = re.compile('--browser=(?P<browser_type>\S+)')
1585 matches = cmd_re.search(command_to_run)
1586 if bisect_utils.IsStringInt(commit_position) and matches:
1587 cmd_browser = matches.group('browser_type')
1588 if commit_position <= 274857 and cmd_browser == 'android-chrome-shell':
1589 return command_to_run.replace(cmd_browser,
1590 'android-chromium-testshell')
1591 elif (commit_position >= 276628 and
1592 cmd_browser == 'android-chromium-testshell'):
1593 return command_to_run.replace(cmd_browser,
1594 'android-chrome-shell')
1595 return command_to_run
1596
1597 def RunPerformanceTestAndParseResults(
1598 self, command_to_run, metric, reset_on_first_run=False,
1599 upload_on_last_run=False, results_label=None):
1600 """Runs a performance test on the current revision and parses the results.
1601
1602 Args:
1603 command_to_run: The command to be run to execute the performance test.
1604 metric: The metric to parse out from the results of the performance test.
1605 This is the result chart name and trace name, separated by slash.
1606 May be None for perf try jobs.
1607 reset_on_first_run: If True, pass the flag --reset-results on first run.
1608 upload_on_last_run: If True, pass the flag --upload-results on last run.
1609 results_label: A value for the option flag --results-label.
1610 The arguments reset_on_first_run, upload_on_last_run and results_label
1611 are all ignored if the test is not a Telemetry test.
1612
1613 Returns:
1614 (values dict, 0) if --debug_ignore_perf_test was passed.
1615 (values dict, 0, test output) if the test was run successfully.
1616 (error message, -1) if the test couldn't be run.
1617 (error message, -1, test output) if the test ran but there was an error.
1618 """
1619 success_code, failure_code = 0, -1
1620
1621 if self.opts.debug_ignore_perf_test:
1622 fake_results = {
1623 'mean': 0.0,
1624 'std_err': 0.0,
1625 'std_dev': 0.0,
1626 'values': [0.0]
1627 }
1628 return (fake_results, success_code)
1629
1630 # For Windows platform set posix=False, to parse windows paths correctly.
1631 # On Windows, path separators '\' or '\\' are replace by '' when posix=True,
1632 # refer to http://bugs.python.org/issue1724822. By default posix=True.
1633 args = shlex.split(command_to_run, posix=not bisect_utils.IsWindowsHost())
1634
1635 if not _GenerateProfileIfNecessary(args):
1636 err_text = 'Failed to generate profile for performance test.'
1637 return (err_text, failure_code)
1638
1639 # If running a Telemetry test for Chrome OS, insert the remote IP and
1640 # identity parameters.
1641 is_telemetry = bisect_utils.IsTelemetryCommand(command_to_run)
1642 if self.opts.target_platform == 'cros' and is_telemetry:
1643 args.append('--remote=%s' % self.opts.cros_remote_ip)
1644 args.append('--identity=%s' % bisect_utils.CROS_TEST_KEY_PATH)
1645
1646 start_time = time.time()
1647
1648 metric_values = []
1649 output_of_all_runs = ''
1650 for i in xrange(self.opts.repeat_test_count):
1651 # Can ignore the return code since if the tests fail, it won't return 0.
1652 current_args = copy.copy(args)
1653 if is_telemetry:
1654 if i == 0 and reset_on_first_run:
1655 current_args.append('--reset-results')
1656 elif i == self.opts.repeat_test_count - 1 and upload_on_last_run:
1657 current_args.append('--upload-results')
1658 if results_label:
1659 current_args.append('--results-label=%s' % results_label)
1660 try:
1661 output, return_code = bisect_utils.RunProcessAndRetrieveOutput(
1662 current_args, cwd=self.src_cwd)
1663 except OSError, e:
1664 if e.errno == errno.ENOENT:
1665 err_text = ('Something went wrong running the performance test. '
1666 'Please review the command line:\n\n')
1667 if 'src/' in ' '.join(args):
1668 err_text += ('Check that you haven\'t accidentally specified a '
1669 'path with src/ in the command.\n\n')
1670 err_text += ' '.join(args)
1671 err_text += '\n'
1672
1673 return (err_text, failure_code)
1674 raise
1675
1676 output_of_all_runs += output
1677 if self.opts.output_buildbot_annotations:
1678 print output
1679
1680 if metric and self._IsBisectModeUsingMetric():
1681 metric_values += _ParseMetricValuesFromOutput(metric, output)
1682 # If we're bisecting on a metric (ie, changes in the mean or
1683 # standard deviation) and no metric values are produced, bail out.
1684 if not metric_values:
1685 break
1686 elif self._IsBisectModeReturnCode():
1687 metric_values.append(return_code)
1688
1689 elapsed_minutes = (time.time() - start_time) / 60.0
1690 if elapsed_minutes >= self.opts.max_time_minutes:
1691 break
1692
1693 if metric and len(metric_values) == 0:
1694 err_text = 'Metric %s was not found in the test output.' % metric
1695 # TODO(qyearsley): Consider also getting and displaying a list of metrics
1696 # that were found in the output here.
1697 return (err_text, failure_code, output_of_all_runs)
1698
1699 # If we're bisecting on return codes, we're really just looking for zero vs
1700 # non-zero.
1701 values = {}
1702 if self._IsBisectModeReturnCode():
1703 # If any of the return codes is non-zero, output 1.
1704 overall_return_code = 0 if (
1705 all(current_value == 0 for current_value in metric_values)) else 1
1706
1707 values = {
1708 'mean': overall_return_code,
1709 'std_err': 0.0,
1710 'std_dev': 0.0,
1711 'values': metric_values,
1712 }
1713
1714 print 'Results of performance test: Command returned with %d' % (
1715 overall_return_code)
1716 print
1717 elif metric:
1718 # Need to get the average value if there were multiple values.
1719 truncated_mean = math_utils.TruncatedMean(
1720 metric_values, self.opts.truncate_percent)
1721 standard_err = math_utils.StandardError(metric_values)
1722 standard_dev = math_utils.StandardDeviation(metric_values)
1723
1724 if self._IsBisectModeStandardDeviation():
1725 metric_values = [standard_dev]
1726
1727 values = {
1728 'mean': truncated_mean,
1729 'std_err': standard_err,
1730 'std_dev': standard_dev,
1731 'values': metric_values,
1732 }
1733
1734 print 'Results of performance test: %12f %12f' % (
1735 truncated_mean, standard_err)
1736 print
1737 return (values, success_code, output_of_all_runs)
1738
1739 def _FindAllRevisionsToSync(self, revision, depot):
1740 """Finds all dependent revisions and depots that need to be synced.
1741
1742 For example skia is broken up into 3 git mirrors over skia/src,
1743 skia/gyp, and skia/include. To sync skia/src properly, one has to find
1744 the proper revisions in skia/gyp and skia/include.
1745
1746 This is only useful in the git workflow, as an SVN depot may be split into
1747 multiple mirrors.
1748
1749 Args:
1750 revision: The revision to sync to.
1751 depot: The depot in use at the moment (probably skia).
1752
1753 Returns:
1754 A list of [depot, revision] pairs that need to be synced.
1755 """
1756 revisions_to_sync = [[depot, revision]]
1757
1758 is_base = ((depot == 'chromium') or (depot == 'cros') or
1759 (depot == 'android-chrome'))
1760
1761 # Some SVN depots were split into multiple git depots, so we need to
1762 # figure out for each mirror which git revision to grab. There's no
1763 # guarantee that the SVN revision will exist for each of the dependent
1764 # depots, so we have to grep the git logs and grab the next earlier one.
1765 if (not is_base
1766 and DEPOT_DEPS_NAME[depot]['depends']
1767 and self.source_control.IsGit()):
1768 commit_position = self.source_control.GetCommitPosition(revision)
1769
1770 for d in DEPOT_DEPS_NAME[depot]['depends']:
1771 self.ChangeToDepotWorkingDirectory(d)
1772
1773 dependant_rev = self.source_control.ResolveToRevision(
1774 commit_position, d, DEPOT_DEPS_NAME, -1000)
1775
1776 if dependant_rev:
1777 revisions_to_sync.append([d, dependant_rev])
1778
1779 num_resolved = len(revisions_to_sync)
1780 num_needed = len(DEPOT_DEPS_NAME[depot]['depends'])
1781
1782 self.ChangeToDepotWorkingDirectory(depot)
1783
1784 if not ((num_resolved - 1) == num_needed):
1785 return None
1786
1787 return revisions_to_sync
1788
1789 def PerformPreBuildCleanup(self):
1790 """Performs cleanup between runs."""
1791 print 'Cleaning up between runs.'
1792 print
1793
1794 # Leaving these .pyc files around between runs may disrupt some perf tests.
1795 for (path, _, files) in os.walk(self.src_cwd):
1796 for cur_file in files:
1797 if cur_file.endswith('.pyc'):
1798 path_to_file = os.path.join(path, cur_file)
1799 os.remove(path_to_file)
1800
1801 def PerformCrosChrootCleanup(self):
1802 """Deletes the chroot.
1803
1804 Returns:
1805 True if successful.
1806 """
1807 cwd = os.getcwd()
1808 self.ChangeToDepotWorkingDirectory('cros')
1809 cmd = [bisect_utils.CROS_SDK_PATH, '--delete']
1810 return_code = bisect_utils.RunProcess(cmd)
1811 os.chdir(cwd)
1812 return not return_code
1813
1814 def CreateCrosChroot(self):
1815 """Creates a new chroot.
1816
1817 Returns:
1818 True if successful.
1819 """
1820 cwd = os.getcwd()
1821 self.ChangeToDepotWorkingDirectory('cros')
1822 cmd = [bisect_utils.CROS_SDK_PATH, '--create']
1823 return_code = bisect_utils.RunProcess(cmd)
1824 os.chdir(cwd)
1825 return not return_code
1826
1827 def _PerformPreSyncCleanup(self, depot):
1828 """Performs any necessary cleanup before syncing.
1829
1830 Args:
1831 depot: Depot name.
1832
1833 Returns:
1834 True if successful.
1835 """
1836 if depot == 'chromium' or depot == 'android-chrome':
1837 # Removes third_party/libjingle. At some point, libjingle was causing
1838 # issues syncing when using the git workflow (crbug.com/266324).
1839 os.chdir(self.src_cwd)
1840 if not bisect_utils.RemoveThirdPartyDirectory('libjingle'):
1841 return False
1842 # Removes third_party/skia. At some point, skia was causing
1843 # issues syncing when using the git workflow (crbug.com/377951).
1844 if not bisect_utils.RemoveThirdPartyDirectory('skia'):
1845 return False
1846 elif depot == 'cros':
1847 return self.PerformCrosChrootCleanup()
1848 return True
1849
1850 def _RunPostSync(self, depot):
1851 """Performs any work after syncing.
1852
1853 Args:
1854 depot: Depot name.
1855
1856 Returns:
1857 True if successful.
1858 """
1859 if self.opts.target_platform == 'android':
1860 if not builder.SetupAndroidBuildEnvironment(self.opts,
1861 path_to_src=self.src_cwd):
1862 return False
1863
1864 if depot == 'cros':
1865 return self.CreateCrosChroot()
1866 else:
1867 return self.RunGClientHooks()
1868 return True
1869
1870 def ShouldSkipRevision(self, depot, revision):
1871 """Checks whether a particular revision can be safely skipped.
1872
1873 Some commits can be safely skipped (such as a DEPS roll), since the tool
1874 is git based those changes would have no effect.
1875
1876 Args:
1877 depot: The depot being bisected.
1878 revision: Current revision we're synced to.
1879
1880 Returns:
1881 True if we should skip building/testing this revision.
1882 """
1883 if depot == 'chromium':
1884 if self.source_control.IsGit():
1885 cmd = ['diff-tree', '--no-commit-id', '--name-only', '-r', revision]
1886 output = bisect_utils.CheckRunGit(cmd)
1887
1888 files = output.splitlines()
1889
1890 if len(files) == 1 and files[0] == 'DEPS':
1891 return True
1892
1893 return False
1894
1895 def RunTest(self, revision, depot, command, metric, skippable=False):
1896 """Performs a full sync/build/run of the specified revision.
1897
1898 Args:
1899 revision: The revision to sync to.
1900 depot: The depot that's being used at the moment (src, webkit, etc.)
1901 command: The command to execute the performance test.
1902 metric: The performance metric being tested.
1903
1904 Returns:
1905 On success, a tuple containing the results of the performance test.
1906 Otherwise, a tuple with the error message.
1907 """
1908 # Decide which sync program to use.
1909 sync_client = None
1910 if depot == 'chromium' or depot == 'android-chrome':
1911 sync_client = 'gclient'
1912 elif depot == 'cros':
1913 sync_client = 'repo'
1914
1915 # Decide what depots will need to be synced to what revisions.
1916 revisions_to_sync = self._FindAllRevisionsToSync(revision, depot)
1917 if not revisions_to_sync:
1918 return ('Failed to resolve dependent depots.', BUILD_RESULT_FAIL)
1919
1920 if not self._PerformPreSyncCleanup(depot):
1921 return ('Failed to perform pre-sync cleanup.', BUILD_RESULT_FAIL)
1922
1923 # Do the syncing for all depots.
1924 if not self.opts.debug_ignore_sync:
1925 if not self._SyncAllRevisions(revisions_to_sync, sync_client):
1926 return ('Failed to sync: [%s]' % str(revision), BUILD_RESULT_FAIL)
1927
1928 # Try to do any post-sync steps. This may include "gclient runhooks".
1929 if not self._RunPostSync(depot):
1930 return ('Failed to run [gclient runhooks].', BUILD_RESULT_FAIL)
1931
1932 # Skip this revision if it can be skipped.
1933 if skippable and self.ShouldSkipRevision(depot, revision):
1934 return ('Skipped revision: [%s]' % str(revision),
1935 BUILD_RESULT_SKIPPED)
1936
1937 # Obtain a build for this revision. This may be done by requesting a build
1938 # from another builder, waiting for it and downloading it.
1939 start_build_time = time.time()
1940 build_success = self.BuildCurrentRevision(depot, revision)
1941 if not build_success:
1942 return ('Failed to build revision: [%s]' % str(revision),
1943 BUILD_RESULT_FAIL)
1944 after_build_time = time.time()
1945
1946 # Possibly alter the command.
1947 command = self.GetCompatibleCommand(command, revision, depot)
1948
1949 # Run the command and get the results.
1950 results = self.RunPerformanceTestAndParseResults(command, metric)
1951
1952 # Restore build output directory once the tests are done, to avoid
1953 # any discrepancies.
1954 if self.IsDownloadable(depot) and revision:
1955 self.BackupOrRestoreOutputDirectory(restore=True)
1956
1957 # A value other than 0 indicates that the test couldn't be run, and results
1958 # should also include an error message.
1959 if results[1] != 0:
1960 return results
1961
1962 external_revisions = self._Get3rdPartyRevisions(depot)
1963
1964 if not external_revisions is None:
1965 return (results[0], results[1], external_revisions,
1966 time.time() - after_build_time, after_build_time -
1967 start_build_time)
1968 else:
1969 return ('Failed to parse DEPS file for external revisions.',
1970 BUILD_RESULT_FAIL)
1971
1972 def _SyncAllRevisions(self, revisions_to_sync, sync_client):
1973 """Syncs multiple depots to particular revisions.
1974
1975 Args:
1976 revisions_to_sync: A list of (depot, revision) pairs to be synced.
1977 sync_client: Program used to sync, e.g. "gclient", "repo". Can be None.
1978
1979 Returns:
1980 True if successful, False otherwise.
1981 """
1982 for depot, revision in revisions_to_sync:
1983 self.ChangeToDepotWorkingDirectory(depot)
1984
1985 if sync_client:
1986 self.PerformPreBuildCleanup()
1987
1988 # When using gclient to sync, you need to specify the depot you
1989 # want so that all the dependencies sync properly as well.
1990 # i.e. gclient sync src@<SHA1>
1991 if sync_client == 'gclient':
1992 revision = '%s@%s' % (DEPOT_DEPS_NAME[depot]['src'], revision)
1993
1994 sync_success = self.source_control.SyncToRevision(revision, sync_client)
1995 if not sync_success:
1996 return False
1997
1998 return True
1999
2000 def _CheckIfRunPassed(self, current_value, known_good_value, known_bad_value):
2001 """Given known good and bad values, decide if the current_value passed
2002 or failed.
2003
2004 Args:
2005 current_value: The value of the metric being checked.
2006 known_bad_value: The reference value for a "failed" run.
2007 known_good_value: The reference value for a "passed" run.
2008
2009 Returns:
2010 True if the current_value is closer to the known_good_value than the
2011 known_bad_value.
2012 """
2013 if self.opts.bisect_mode == BISECT_MODE_STD_DEV:
2014 dist_to_good_value = abs(current_value['std_dev'] -
2015 known_good_value['std_dev'])
2016 dist_to_bad_value = abs(current_value['std_dev'] -
2017 known_bad_value['std_dev'])
2018 else:
2019 dist_to_good_value = abs(current_value['mean'] - known_good_value['mean'])
2020 dist_to_bad_value = abs(current_value['mean'] - known_bad_value['mean'])
2021
2022 return dist_to_good_value < dist_to_bad_value
2023
2024 def _GetDepotDirectory(self, depot_name):
2025 if depot_name == 'chromium':
2026 return self.src_cwd
2027 elif depot_name == 'cros':
2028 return self.cros_cwd
2029 elif depot_name in DEPOT_NAMES:
2030 return self.depot_cwd[depot_name]
2031 else:
2032 assert False, ('Unknown depot [ %s ] encountered. Possibly a new one '
2033 'was added without proper support?' % depot_name)
2034
2035 def ChangeToDepotWorkingDirectory(self, depot_name):
2036 """Given a depot, changes to the appropriate working directory.
2037
2038 Args:
2039 depot_name: The name of the depot (see DEPOT_NAMES).
2040 """
2041 os.chdir(self._GetDepotDirectory(depot_name))
2042
2043 def _FillInV8BleedingEdgeInfo(self, min_revision_data, max_revision_data):
2044 r1 = self._GetNearestV8BleedingEdgeFromTrunk(min_revision_data['revision'],
2045 search_forward=True)
2046 r2 = self._GetNearestV8BleedingEdgeFromTrunk(max_revision_data['revision'],
2047 search_forward=False)
2048 min_revision_data['external']['v8_bleeding_edge'] = r1
2049 max_revision_data['external']['v8_bleeding_edge'] = r2
2050
2051 if (not self._GetV8BleedingEdgeFromV8TrunkIfMappable(
2052 min_revision_data['revision'])
2053 or not self._GetV8BleedingEdgeFromV8TrunkIfMappable(
2054 max_revision_data['revision'])):
2055 self.warnings.append(
2056 'Trunk revisions in V8 did not map directly to bleeding_edge. '
2057 'Attempted to expand the range to find V8 rolls which did map '
2058 'directly to bleeding_edge revisions, but results might not be '
2059 'valid.')
2060
2061 def _FindNextDepotToBisect(
2062 self, current_depot, min_revision_data, max_revision_data):
2063 """Decides which depot the script should dive into next (if any).
2064
2065 Args:
2066 current_depot: Current depot being bisected.
2067 min_revision_data: Data about the earliest revision in the bisect range.
2068 max_revision_data: Data about the latest revision in the bisect range.
2069
2070 Returns:
2071 Name of the depot to bisect next, or None.
2072 """
2073 external_depot = None
2074 for next_depot in DEPOT_NAMES:
2075 if DEPOT_DEPS_NAME[next_depot].has_key('platform'):
2076 if DEPOT_DEPS_NAME[next_depot]['platform'] != os.name:
2077 continue
2078
2079 if not (DEPOT_DEPS_NAME[next_depot]['recurse']
2080 and min_revision_data['depot']
2081 in DEPOT_DEPS_NAME[next_depot]['from']):
2082 continue
2083
2084 if current_depot == 'v8':
2085 # We grab the bleeding_edge info here rather than earlier because we
2086 # finally have the revision range. From that we can search forwards and
2087 # backwards to try to match trunk revisions to bleeding_edge.
2088 self._FillInV8BleedingEdgeInfo(min_revision_data, max_revision_data)
2089
2090 if (min_revision_data['external'].get(next_depot) ==
2091 max_revision_data['external'].get(next_depot)):
2092 continue
2093
2094 if (min_revision_data['external'].get(next_depot) and
2095 max_revision_data['external'].get(next_depot)):
2096 external_depot = next_depot
2097 break
2098
2099 return external_depot
2100
2101 def PrepareToBisectOnDepot(
2102 self, current_depot, end_revision, start_revision, previous_revision):
2103 """Changes to the appropriate directory and gathers a list of revisions
2104 to bisect between |start_revision| and |end_revision|.
2105
2106 Args:
2107 current_depot: The depot we want to bisect.
2108 end_revision: End of the revision range.
2109 start_revision: Start of the revision range.
2110 previous_revision: The last revision we synced to on |previous_depot|.
2111
2112 Returns:
2113 A list containing the revisions between |start_revision| and
2114 |end_revision| inclusive.
2115 """
2116 # Change into working directory of external library to run
2117 # subsequent commands.
2118 self.ChangeToDepotWorkingDirectory(current_depot)
2119
2120 # V8 (and possibly others) is merged in periodically. Bisecting
2121 # this directory directly won't give much good info.
2122 if DEPOT_DEPS_NAME[current_depot].has_key('custom_deps'):
2123 config_path = os.path.join(self.src_cwd, '..')
2124 if bisect_utils.RunGClientAndCreateConfig(self.opts,
2125 DEPOT_DEPS_NAME[current_depot]['custom_deps'], cwd=config_path):
2126 return []
2127 if bisect_utils.RunGClient(
2128 ['sync', '--revision', previous_revision], cwd=self.src_cwd):
2129 return []
2130
2131 if current_depot == 'v8_bleeding_edge':
2132 self.ChangeToDepotWorkingDirectory('chromium')
2133
2134 shutil.move('v8', 'v8.bak')
2135 shutil.move('v8_bleeding_edge', 'v8')
2136
2137 self.cleanup_commands.append(['mv', 'v8', 'v8_bleeding_edge'])
2138 self.cleanup_commands.append(['mv', 'v8.bak', 'v8'])
2139
2140 self.depot_cwd['v8_bleeding_edge'] = os.path.join(self.src_cwd, 'v8')
2141 self.depot_cwd['v8'] = os.path.join(self.src_cwd, 'v8.bak')
2142
2143 self.ChangeToDepotWorkingDirectory(current_depot)
2144
2145 depot_revision_list = self.GetRevisionList(current_depot,
2146 end_revision,
2147 start_revision)
2148
2149 self.ChangeToDepotWorkingDirectory('chromium')
2150
2151 return depot_revision_list
2152
2153 def GatherReferenceValues(self, good_rev, bad_rev, cmd, metric, target_depot):
2154 """Gathers reference values by running the performance tests on the
2155 known good and bad revisions.
2156
2157 Args:
2158 good_rev: The last known good revision where the performance regression
2159 has not occurred yet.
2160 bad_rev: A revision where the performance regression has already occurred.
2161 cmd: The command to execute the performance test.
2162 metric: The metric being tested for regression.
2163
2164 Returns:
2165 A tuple with the results of building and running each revision.
2166 """
2167 bad_run_results = self.RunTest(bad_rev, target_depot, cmd, metric)
2168
2169 good_run_results = None
2170
2171 if not bad_run_results[1]:
2172 good_run_results = self.RunTest(good_rev, target_depot, cmd, metric)
2173
2174 return (bad_run_results, good_run_results)
2175
2176 def PrintRevisionsToBisectMessage(self, revision_list, depot):
2177 if self.opts.output_buildbot_annotations:
2178 step_name = 'Bisection Range: [%s - %s]' % (
2179 revision_list[len(revision_list)-1], revision_list[0])
2180 bisect_utils.OutputAnnotationStepStart(step_name)
2181
2182 print
2183 print 'Revisions to bisect on [%s]:' % depot
2184 for revision_id in revision_list:
2185 print ' -> %s' % (revision_id, )
2186 print
2187
2188 if self.opts.output_buildbot_annotations:
2189 bisect_utils.OutputAnnotationStepClosed()
2190
2191 def NudgeRevisionsIfDEPSChange(self, bad_revision, good_revision,
2192 good_svn_revision=None):
2193 """Checks to see if changes to DEPS file occurred, and that the revision
2194 range also includes the change to .DEPS.git. If it doesn't, attempts to
2195 expand the revision range to include it.
2196
2197 Args:
2198 bad_revision: First known bad git revision.
2199 good_revision: Last known good git revision.
2200 good_svn_revision: Last known good svn revision.
2201
2202 Returns:
2203 A tuple with the new bad and good revisions.
2204 """
2205 # DONOT perform nudge because at revision 291563 .DEPS.git was removed
2206 # and source contain only DEPS file for dependency changes.
2207 if good_svn_revision >= 291563:
2208 return (bad_revision, good_revision)
2209
2210 if self.source_control.IsGit() and self.opts.target_platform == 'chromium':
2211 changes_to_deps = self.source_control.QueryFileRevisionHistory(
2212 bisect_utils.FILE_DEPS, good_revision, bad_revision)
2213
2214 if changes_to_deps:
2215 # DEPS file was changed, search from the oldest change to DEPS file to
2216 # bad_revision to see if there are matching .DEPS.git changes.
2217 oldest_deps_change = changes_to_deps[-1]
2218 changes_to_gitdeps = self.source_control.QueryFileRevisionHistory(
2219 bisect_utils.FILE_DEPS_GIT, oldest_deps_change, bad_revision)
2220
2221 if len(changes_to_deps) != len(changes_to_gitdeps):
2222 # Grab the timestamp of the last DEPS change
2223 cmd = ['log', '--format=%ct', '-1', changes_to_deps[0]]
2224 output = bisect_utils.CheckRunGit(cmd)
2225 commit_time = int(output)
2226
2227 # Try looking for a commit that touches the .DEPS.git file in the
2228 # next 15 minutes after the DEPS file change.
2229 cmd = ['log', '--format=%H', '-1',
2230 '--before=%d' % (commit_time + 900), '--after=%d' % commit_time,
2231 'origin/master', '--', bisect_utils.FILE_DEPS_GIT]
2232 output = bisect_utils.CheckRunGit(cmd)
2233 output = output.strip()
2234 if output:
2235 self.warnings.append('Detected change to DEPS and modified '
2236 'revision range to include change to .DEPS.git')
2237 return (output, good_revision)
2238 else:
2239 self.warnings.append('Detected change to DEPS but couldn\'t find '
2240 'matching change to .DEPS.git')
2241 return (bad_revision, good_revision)
2242
2243 def CheckIfRevisionsInProperOrder(
2244 self, target_depot, good_revision, bad_revision):
2245 """Checks that |good_revision| is an earlier revision than |bad_revision|.
2246
2247 Args:
2248 good_revision: Number/tag of the known good revision.
2249 bad_revision: Number/tag of the known bad revision.
2250
2251 Returns:
2252 True if the revisions are in the proper order (good earlier than bad).
2253 """
2254 if self.source_control.IsGit() and target_depot != 'cros':
2255 cwd = self._GetDepotDirectory(target_depot)
2256
2257 cmd = ['log', '--format=%ct', '-1', good_revision]
2258 output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
2259 good_commit_time = int(output)
2260
2261 cmd = ['log', '--format=%ct', '-1', bad_revision]
2262 output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
2263 bad_commit_time = int(output)
2264
2265 return good_commit_time <= bad_commit_time
2266 else:
2267 # CrOS and SVN use integers.
2268 return int(good_revision) <= int(bad_revision)
2269
2270 def CanPerformBisect(self, good_revision, bad_revision):
2271 """Checks whether a given revision is bisectable.
2272
2273 Checks for following:
2274 1. Non-bisectable revsions for android bots (refer to crbug.com/385324).
2275 2. Non-bisectable revsions for Windows bots (refer to crbug.com/405274).
2276
2277 Args:
2278 good_revision: Known good revision.
2279 bad_revision: Known bad revision.
2280
2281 Returns:
2282 A dictionary indicating the result. If revision is not bisectable,
2283 this will contain the field "error", otherwise None.
2284 """
2285 if self.opts.target_platform == 'android':
2286 revision_to_check = self.source_control.GetCommitPosition(good_revision)
2287 if (bisect_utils.IsStringInt(good_revision)
2288 and good_revision < 265549):
2289 return {'error': (
2290 'Bisect cannot continue for the given revision range.\n'
2291 'It is impossible to bisect Android regressions '
2292 'prior to r265549, which allows the bisect bot to '
2293 'rely on Telemetry to do apk installation of the most recently '
2294 'built local ChromeShell(refer to crbug.com/385324).\n'
2295 'Please try bisecting revisions greater than or equal to r265549.')}
2296
2297 if bisect_utils.IsWindowsHost():
2298 good_revision = self.source_control.GetCommitPosition(good_revision)
2299 bad_revision = self.source_control.GetCommitPosition(bad_revision)
2300 if (bisect_utils.IsStringInt(good_revision) and
2301 bisect_utils.IsStringInt(bad_revision)):
2302 if (289987 <= good_revision < 290716 or
2303 289987 <= bad_revision < 290716):
2304 return {'error': ('Oops! Revision between r289987 and r290716 are '
2305 'marked as dead zone for Windows due to '
2306 'crbug.com/405274. Please try another range.')}
2307
2308 return None
2309
2310 def Run(self, command_to_run, bad_revision_in, good_revision_in, metric):
2311 """Given known good and bad revisions, run a binary search on all
2312 intermediate revisions to determine the CL where the performance regression
2313 occurred.
2314
2315 Args:
2316 command_to_run: Specify the command to execute the performance test.
2317 good_revision: Number/tag of the known good revision.
2318 bad_revision: Number/tag of the known bad revision.
2319 metric: The performance metric to monitor.
2320
2321 Returns:
2322 A dict with 2 members, 'revision_data' and 'error'. On success,
2323 'revision_data' will contain a dict mapping revision ids to
2324 data about that revision. Each piece of revision data consists of a
2325 dict with the following keys:
2326
2327 'passed': Represents whether the performance test was successful at
2328 that revision. Possible values include: 1 (passed), 0 (failed),
2329 '?' (skipped), 'F' (build failed).
2330 'depot': The depot that this revision is from (i.e. WebKit)
2331 'external': If the revision is a 'src' revision, 'external' contains
2332 the revisions of each of the external libraries.
2333 'sort': A sort value for sorting the dict in order of commits.
2334
2335 For example:
2336 {
2337 'error':None,
2338 'revision_data':
2339 {
2340 'CL #1':
2341 {
2342 'passed': False,
2343 'depot': 'chromium',
2344 'external': None,
2345 'sort': 0
2346 }
2347 }
2348 }
2349
2350 If an error occurred, the 'error' field will contain the message and
2351 'revision_data' will be empty.
2352 """
2353 results = {
2354 'revision_data' : {},
2355 'error' : None,
2356 }
2357
2358 # Choose depot to bisect first
2359 target_depot = 'chromium'
2360 if self.opts.target_platform == 'cros':
2361 target_depot = 'cros'
2362 elif self.opts.target_platform == 'android-chrome':
2363 target_depot = 'android-chrome'
2364
2365 cwd = os.getcwd()
2366 self.ChangeToDepotWorkingDirectory(target_depot)
2367
2368 # If they passed SVN revisions, we can try match them to git SHA1 hashes.
2369 bad_revision = self.source_control.ResolveToRevision(
2370 bad_revision_in, target_depot, DEPOT_DEPS_NAME, 100)
2371 good_revision = self.source_control.ResolveToRevision(
2372 good_revision_in, target_depot, DEPOT_DEPS_NAME, -100)
2373
2374 os.chdir(cwd)
2375 if bad_revision is None:
2376 results['error'] = 'Couldn\'t resolve [%s] to SHA1.' % bad_revision_in
2377 return results
2378
2379 if good_revision is None:
2380 results['error'] = 'Couldn\'t resolve [%s] to SHA1.' % good_revision_in
2381 return results
2382
2383 # Check that they didn't accidentally swap good and bad revisions.
2384 if not self.CheckIfRevisionsInProperOrder(
2385 target_depot, good_revision, bad_revision):
2386 results['error'] = ('bad_revision < good_revision, did you swap these '
2387 'by mistake?')
2388 return results
2389 bad_revision, good_revision = self.NudgeRevisionsIfDEPSChange(
2390 bad_revision, good_revision, good_revision_in)
2391 if self.opts.output_buildbot_annotations:
2392 bisect_utils.OutputAnnotationStepStart('Gathering Revisions')
2393
2394 cannot_bisect = self.CanPerformBisect(good_revision, bad_revision)
2395 if cannot_bisect:
2396 results['error'] = cannot_bisect.get('error')
2397 return results
2398
2399 print 'Gathering revision range for bisection.'
2400 # Retrieve a list of revisions to do bisection on.
2401 src_revision_list = self.GetRevisionList(
2402 target_depot, bad_revision, good_revision)
2403
2404 if self.opts.output_buildbot_annotations:
2405 bisect_utils.OutputAnnotationStepClosed()
2406
2407 if src_revision_list:
2408 # revision_data will store information about a revision such as the
2409 # depot it came from, the webkit/V8 revision at that time,
2410 # performance timing, build state, etc...
2411 revision_data = results['revision_data']
2412
2413 # revision_list is the list we're binary searching through at the moment.
2414 revision_list = []
2415
2416 sort_key_ids = 0
2417
2418 for current_revision_id in src_revision_list:
2419 sort_key_ids += 1
2420
2421 revision_data[current_revision_id] = {
2422 'value' : None,
2423 'passed' : '?',
2424 'depot' : target_depot,
2425 'external' : None,
2426 'perf_time' : 0,
2427 'build_time' : 0,
2428 'sort' : sort_key_ids,
2429 }
2430 revision_list.append(current_revision_id)
2431
2432 min_revision = 0
2433 max_revision = len(revision_list) - 1
2434
2435 self.PrintRevisionsToBisectMessage(revision_list, target_depot)
2436
2437 if self.opts.output_buildbot_annotations:
2438 bisect_utils.OutputAnnotationStepStart('Gathering Reference Values')
2439
2440 print 'Gathering reference values for bisection.'
2441
2442 # Perform the performance tests on the good and bad revisions, to get
2443 # reference values.
2444 bad_results, good_results = self.GatherReferenceValues(good_revision,
2445 bad_revision,
2446 command_to_run,
2447 metric,
2448 target_depot)
2449
2450 if self.opts.output_buildbot_annotations:
2451 bisect_utils.OutputAnnotationStepClosed()
2452
2453 if bad_results[1]:
2454 results['error'] = ('An error occurred while building and running '
2455 'the \'bad\' reference value. The bisect cannot continue without '
2456 'a working \'bad\' revision to start from.\n\nError: %s' %
2457 bad_results[0])
2458 return results
2459
2460 if good_results[1]:
2461 results['error'] = ('An error occurred while building and running '
2462 'the \'good\' reference value. The bisect cannot continue without '
2463 'a working \'good\' revision to start from.\n\nError: %s' %
2464 good_results[0])
2465 return results
2466
2467
2468 # We need these reference values to determine if later runs should be
2469 # classified as pass or fail.
2470 known_bad_value = bad_results[0]
2471 known_good_value = good_results[0]
2472
2473 # Can just mark the good and bad revisions explicitly here since we
2474 # already know the results.
2475 bad_revision_data = revision_data[revision_list[0]]
2476 bad_revision_data['external'] = bad_results[2]
2477 bad_revision_data['perf_time'] = bad_results[3]
2478 bad_revision_data['build_time'] = bad_results[4]
2479 bad_revision_data['passed'] = False
2480 bad_revision_data['value'] = known_bad_value
2481
2482 good_revision_data = revision_data[revision_list[max_revision]]
2483 good_revision_data['external'] = good_results[2]
2484 good_revision_data['perf_time'] = good_results[3]
2485 good_revision_data['build_time'] = good_results[4]
2486 good_revision_data['passed'] = True
2487 good_revision_data['value'] = known_good_value
2488
2489 next_revision_depot = target_depot
2490
2491 while True:
2492 if not revision_list:
2493 break
2494
2495 min_revision_data = revision_data[revision_list[min_revision]]
2496 max_revision_data = revision_data[revision_list[max_revision]]
2497
2498 if max_revision - min_revision <= 1:
2499 current_depot = min_revision_data['depot']
2500 if min_revision_data['passed'] == '?':
2501 next_revision_index = min_revision
2502 elif max_revision_data['passed'] == '?':
2503 next_revision_index = max_revision
2504 elif current_depot in ['android-chrome', 'cros', 'chromium', 'v8']:
2505 previous_revision = revision_list[min_revision]
2506 # If there were changes to any of the external libraries we track,
2507 # should bisect the changes there as well.
2508 external_depot = self._FindNextDepotToBisect(
2509 current_depot, min_revision_data, max_revision_data)
2510 # If there was no change in any of the external depots, the search
2511 # is over.
2512 if not external_depot:
2513 if current_depot == 'v8':
2514 self.warnings.append('Unfortunately, V8 bisection couldn\'t '
2515 'continue any further. The script can only bisect into '
2516 'V8\'s bleeding_edge repository if both the current and '
2517 'previous revisions in trunk map directly to revisions in '
2518 'bleeding_edge.')
2519 break
2520
2521 earliest_revision = max_revision_data['external'][external_depot]
2522 latest_revision = min_revision_data['external'][external_depot]
2523
2524 new_revision_list = self.PrepareToBisectOnDepot(
2525 external_depot, latest_revision, earliest_revision,
2526 previous_revision)
2527
2528 if not new_revision_list:
2529 results['error'] = ('An error occurred attempting to retrieve '
2530 'revision range: [%s..%s]' %
2531 (earliest_revision, latest_revision))
2532 return results
2533
2534 _AddRevisionsIntoRevisionData(
2535 new_revision_list, external_depot, min_revision_data['sort'],
2536 revision_data)
2537
2538 # Reset the bisection and perform it on the newly inserted
2539 # changelists.
2540 revision_list = new_revision_list
2541 min_revision = 0
2542 max_revision = len(revision_list) - 1
2543 sort_key_ids += len(revision_list)
2544
2545 print ('Regression in metric %s appears to be the result of '
2546 'changes in [%s].' % (metric, external_depot))
2547
2548 self.PrintRevisionsToBisectMessage(revision_list, external_depot)
2549
2550 continue
2551 else:
2552 break
2553 else:
2554 next_revision_index = (int((max_revision - min_revision) / 2) +
2555 min_revision)
2556
2557 next_revision_id = revision_list[next_revision_index]
2558 next_revision_data = revision_data[next_revision_id]
2559 next_revision_depot = next_revision_data['depot']
2560
2561 self.ChangeToDepotWorkingDirectory(next_revision_depot)
2562
2563 if self.opts.output_buildbot_annotations:
2564 step_name = 'Working on [%s]' % next_revision_id
2565 bisect_utils.OutputAnnotationStepStart(step_name)
2566
2567 print 'Working on revision: [%s]' % next_revision_id
2568
2569 run_results = self.RunTest(
2570 next_revision_id, next_revision_depot, command_to_run, metric,
2571 skippable=True)
2572
2573 # If the build is successful, check whether or not the metric
2574 # had regressed.
2575 if not run_results[1]:
2576 if len(run_results) > 2:
2577 next_revision_data['external'] = run_results[2]
2578 next_revision_data['perf_time'] = run_results[3]
2579 next_revision_data['build_time'] = run_results[4]
2580
2581 passed_regression = self._CheckIfRunPassed(run_results[0],
2582 known_good_value,
2583 known_bad_value)
2584
2585 next_revision_data['passed'] = passed_regression
2586 next_revision_data['value'] = run_results[0]
2587
2588 if passed_regression:
2589 max_revision = next_revision_index
2590 else:
2591 min_revision = next_revision_index
2592 else:
2593 if run_results[1] == BUILD_RESULT_SKIPPED:
2594 next_revision_data['passed'] = 'Skipped'
2595 elif run_results[1] == BUILD_RESULT_FAIL:
2596 next_revision_data['passed'] = 'Build Failed'
2597
2598 print run_results[0]
2599
2600 # If the build is broken, remove it and redo search.
2601 revision_list.pop(next_revision_index)
2602
2603 max_revision -= 1
2604
2605 if self.opts.output_buildbot_annotations:
2606 self._PrintPartialResults(results)
2607 bisect_utils.OutputAnnotationStepClosed()
2608 else:
2609 # Weren't able to sync and retrieve the revision range.
2610 results['error'] = ('An error occurred attempting to retrieve revision '
2611 'range: [%s..%s]' % (good_revision, bad_revision))
2612
2613 return results
2614
2615 def _PrintPartialResults(self, results_dict):
2616 revision_data = results_dict['revision_data']
2617 revision_data_sorted = sorted(revision_data.iteritems(),
2618 key = lambda x: x[1]['sort'])
2619 results_dict = self._GetResultsDict(revision_data, revision_data_sorted)
2620
2621 self._PrintTestedCommitsTable(revision_data_sorted,
2622 results_dict['first_working_revision'],
2623 results_dict['last_broken_revision'],
2624 100, final_step=False)
2625
2626 def _ConfidenceLevelStatus(self, results_dict):
2627 if not results_dict['confidence']:
2628 return None
2629 confidence_status = 'Successful with %(level)s confidence%(warning)s.'
2630 if results_dict['confidence'] >= HIGH_CONFIDENCE:
2631 level = 'high'
2632 else:
2633 level = 'low'
2634 warning = ' and warnings'
2635 if not self.warnings:
2636 warning = ''
2637 return confidence_status % {'level': level, 'warning': warning}
2638
2639 def _GetViewVCLinkFromDepotAndHash(self, cl, depot):
2640 info = self.source_control.QueryRevisionInfo(cl,
2641 self._GetDepotDirectory(depot))
2642 if depot and DEPOT_DEPS_NAME[depot].has_key('viewvc'):
2643 try:
2644 # Format is "git-svn-id: svn://....@123456 <other data>"
2645 svn_line = [i for i in info['body'].splitlines() if 'git-svn-id:' in i]
2646 svn_revision = svn_line[0].split('@')
2647 svn_revision = svn_revision[1].split(' ')[0]
2648 return DEPOT_DEPS_NAME[depot]['viewvc'] + svn_revision
2649 except IndexError:
2650 return ''
2651 return ''
2652
2653 def _PrintRevisionInfo(self, cl, info, depot=None):
2654 email_info = ''
2655 if not info['email'].startswith(info['author']):
2656 email_info = '\nEmail : %s' % info['email']
2657 commit_link = self._GetViewVCLinkFromDepotAndHash(cl, depot)
2658 if commit_link:
2659 commit_info = '\nLink : %s' % commit_link
2660 else:
2661 commit_info = ('\nFailed to parse SVN revision from body:\n%s' %
2662 info['body'])
2663 print RESULTS_REVISION_INFO % {
2664 'subject': info['subject'],
2665 'author': info['author'],
2666 'email_info': email_info,
2667 'commit_info': commit_info,
2668 'cl': cl,
2669 'cl_date': info['date']
2670 }
2671
2672 def _PrintTestedCommitsHeader(self):
2673 if self.opts.bisect_mode == BISECT_MODE_MEAN:
2674 _PrintTableRow(
2675 [20, 70, 14, 12, 13],
2676 ['Depot', 'Commit SHA', 'Mean', 'Std. Error', 'State'])
2677 elif self.opts.bisect_mode == BISECT_MODE_STD_DEV:
2678 _PrintTableRow(
2679 [20, 70, 14, 12, 13],
2680 ['Depot', 'Commit SHA', 'Std. Error', 'Mean', 'State'])
2681 elif self.opts.bisect_mode == BISECT_MODE_RETURN_CODE:
2682 _PrintTableRow(
2683 [20, 70, 14, 13],
2684 ['Depot', 'Commit SHA', 'Return Code', 'State'])
2685 else:
2686 assert False, 'Invalid bisect_mode specified.'
2687
2688 def _PrintTestedCommitsEntry(self, current_data, cl_link, state_str):
2689 if self.opts.bisect_mode == BISECT_MODE_MEAN:
2690 std_error = '+-%.02f' % current_data['value']['std_err']
2691 mean = '%.02f' % current_data['value']['mean']
2692 _PrintTableRow(
2693 [20, 70, 12, 14, 13],
2694 [current_data['depot'], cl_link, mean, std_error, state_str])
2695 elif self.opts.bisect_mode == BISECT_MODE_STD_DEV:
2696 std_error = '+-%.02f' % current_data['value']['std_err']
2697 mean = '%.02f' % current_data['value']['mean']
2698 _PrintTableRow(
2699 [20, 70, 12, 14, 13],
2700 [current_data['depot'], cl_link, std_error, mean, state_str])
2701 elif self.opts.bisect_mode == BISECT_MODE_RETURN_CODE:
2702 mean = '%d' % current_data['value']['mean']
2703 _PrintTableRow(
2704 [20, 70, 14, 13],
2705 [current_data['depot'], cl_link, mean, state_str])
2706
2707 def _PrintTestedCommitsTable(
2708 self, revision_data_sorted, first_working_revision, last_broken_revision,
2709 confidence, final_step=True):
2710 print
2711 if final_step:
2712 print '===== TESTED COMMITS ====='
2713 else:
2714 print '===== PARTIAL RESULTS ====='
2715 self._PrintTestedCommitsHeader()
2716 state = 0
2717 for current_id, current_data in revision_data_sorted:
2718 if current_data['value']:
2719 if (current_id == last_broken_revision or
2720 current_id == first_working_revision):
2721 # If confidence is too low, don't add this empty line since it's
2722 # used to put focus on a suspected CL.
2723 if confidence and final_step:
2724 print
2725 state += 1
2726 if state == 2 and not final_step:
2727 # Just want a separation between "bad" and "good" cl's.
2728 print
2729
2730 state_str = 'Bad'
2731 if state == 1 and final_step:
2732 state_str = 'Suspected CL'
2733 elif state == 2:
2734 state_str = 'Good'
2735
2736 # If confidence is too low, don't bother outputting good/bad.
2737 if not confidence:
2738 state_str = ''
2739 state_str = state_str.center(13, ' ')
2740
2741 cl_link = self._GetViewVCLinkFromDepotAndHash(current_id,
2742 current_data['depot'])
2743 if not cl_link:
2744 cl_link = current_id
2745 self._PrintTestedCommitsEntry(current_data, cl_link, state_str)
2746
2747 def _PrintReproSteps(self):
2748 """Prints out a section of the results explaining how to run the test.
2749
2750 This message includes the command used to run the test.
2751 """
2752 command = '$ ' + self.opts.command
2753 if bisect_utils.IsTelemetryCommand(self.opts.command):
2754 command += ('\nAlso consider passing --profiler=list to see available '
2755 'profilers.')
2756 print REPRO_STEPS_LOCAL
2757 if bisect_utils.IsTelemetryCommand(self.opts.command):
2758 telemetry_command = re.sub(r'--browser=[^\s]+',
2759 '--browser=<bot-name>',
2760 command)
2761 print REPRO_STEPS_TRYJOB_TELEMETRY % {'command': telemetry_command}
2762 else:
2763 print REPRO_STEPS_TRYJOB
2764
2765 def _PrintOtherRegressions(self, other_regressions, revision_data):
2766 """Prints a section of the results about other potential regressions."""
2767 print
2768 print 'Other regressions may have occurred:'
2769 print ' %8s %70s %10s' % ('Depot'.center(8, ' '),
2770 'Range'.center(70, ' '), 'Confidence'.center(10, ' '))
2771 for regression in other_regressions:
2772 current_id, previous_id, confidence = regression
2773 current_data = revision_data[current_id]
2774 previous_data = revision_data[previous_id]
2775
2776 current_link = self._GetViewVCLinkFromDepotAndHash(current_id,
2777 current_data['depot'])
2778 previous_link = self._GetViewVCLinkFromDepotAndHash(previous_id,
2779 previous_data['depot'])
2780
2781 # If we can't map it to a viewable URL, at least show the original hash.
2782 if not current_link:
2783 current_link = current_id
2784 if not previous_link:
2785 previous_link = previous_id
2786
2787 print ' %8s %70s %s' % (
2788 current_data['depot'], current_link,
2789 ('%d%%' % confidence).center(10, ' '))
2790 print ' %8s %70s' % (
2791 previous_data['depot'], previous_link)
2792 print
2793
2794 def _GetResultsDict(self, revision_data, revision_data_sorted):
2795 # Find range where it possibly broke.
2796 first_working_revision = None
2797 first_working_revision_index = -1
2798 last_broken_revision = None
2799 last_broken_revision_index = -1
2800
2801 culprit_revisions = []
2802 other_regressions = []
2803 regression_size = 0.0
2804 regression_std_err = 0.0
2805 confidence = 0.0
2806
2807 for i in xrange(len(revision_data_sorted)):
2808 k, v = revision_data_sorted[i]
2809 if v['passed'] == 1:
2810 if not first_working_revision:
2811 first_working_revision = k
2812 first_working_revision_index = i
2813
2814 if not v['passed']:
2815 last_broken_revision = k
2816 last_broken_revision_index = i
2817
2818 if last_broken_revision != None and first_working_revision != None:
2819 broken_means = []
2820 for i in xrange(0, last_broken_revision_index + 1):
2821 if revision_data_sorted[i][1]['value']:
2822 broken_means.append(revision_data_sorted[i][1]['value']['values'])
2823
2824 working_means = []
2825 for i in xrange(first_working_revision_index, len(revision_data_sorted)):
2826 if revision_data_sorted[i][1]['value']:
2827 working_means.append(revision_data_sorted[i][1]['value']['values'])
2828
2829 # Flatten the lists to calculate mean of all values.
2830 working_mean = sum(working_means, [])
2831 broken_mean = sum(broken_means, [])
2832
2833 # Calculate the approximate size of the regression
2834 mean_of_bad_runs = math_utils.Mean(broken_mean)
2835 mean_of_good_runs = math_utils.Mean(working_mean)
2836
2837 regression_size = 100 * math_utils.RelativeChange(mean_of_good_runs,
2838 mean_of_bad_runs)
2839 if math.isnan(regression_size):
2840 regression_size = 'zero-to-nonzero'
2841
2842 regression_std_err = math.fabs(math_utils.PooledStandardError(
2843 [working_mean, broken_mean]) /
2844 max(0.0001, min(mean_of_good_runs, mean_of_bad_runs))) * 100.0
2845
2846 # Give a "confidence" in the bisect. At the moment we use how distinct the
2847 # values are before and after the last broken revision, and how noisy the
2848 # overall graph is.
2849 confidence = ConfidenceScore(working_means, broken_means)
2850
2851 culprit_revisions = []
2852
2853 cwd = os.getcwd()
2854 self.ChangeToDepotWorkingDirectory(
2855 revision_data[last_broken_revision]['depot'])
2856
2857 if revision_data[last_broken_revision]['depot'] == 'cros':
2858 # Want to get a list of all the commits and what depots they belong
2859 # to so that we can grab info about each.
2860 cmd = ['repo', 'forall', '-c',
2861 'pwd ; git log --pretty=oneline --before=%d --after=%d' % (
2862 last_broken_revision, first_working_revision + 1)]
2863 output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd)
2864
2865 changes = []
2866 assert not return_code, ('An error occurred while running '
2867 '"%s"' % ' '.join(cmd))
2868 last_depot = None
2869 cwd = os.getcwd()
2870 for l in output.split('\n'):
2871 if l:
2872 # Output will be in form:
2873 # /path_to_depot
2874 # /path_to_other_depot
2875 # <SHA1>
2876 # /path_again
2877 # <SHA1>
2878 # etc.
2879 if l[0] == '/':
2880 last_depot = l
2881 else:
2882 contents = l.split(' ')
2883 if len(contents) > 1:
2884 changes.append([last_depot, contents[0]])
2885 for c in changes:
2886 os.chdir(c[0])
2887 info = self.source_control.QueryRevisionInfo(c[1])
2888 culprit_revisions.append((c[1], info, None))
2889 else:
2890 for i in xrange(last_broken_revision_index, len(revision_data_sorted)):
2891 k, v = revision_data_sorted[i]
2892 if k == first_working_revision:
2893 break
2894 self.ChangeToDepotWorkingDirectory(v['depot'])
2895 info = self.source_control.QueryRevisionInfo(k)
2896 culprit_revisions.append((k, info, v['depot']))
2897 os.chdir(cwd)
2898
2899 # Check for any other possible regression ranges.
2900 other_regressions = _FindOtherRegressions(
2901 revision_data_sorted, mean_of_bad_runs > mean_of_good_runs)
2902
2903 return {
2904 'first_working_revision': first_working_revision,
2905 'last_broken_revision': last_broken_revision,
2906 'culprit_revisions': culprit_revisions,
2907 'other_regressions': other_regressions,
2908 'regression_size': regression_size,
2909 'regression_std_err': regression_std_err,
2910 'confidence': confidence,
2911 }
2912
2913 def _CheckForWarnings(self, results_dict):
2914 if len(results_dict['culprit_revisions']) > 1:
2915 self.warnings.append('Due to build errors, regression range could '
2916 'not be narrowed down to a single commit.')
2917 if self.opts.repeat_test_count == 1:
2918 self.warnings.append('Tests were only set to run once. This may '
2919 'be insufficient to get meaningful results.')
2920 if 0 < results_dict['confidence'] < HIGH_CONFIDENCE:
2921 self.warnings.append('Confidence is not high. Try bisecting again '
2922 'with increased repeat_count, larger range, or '
2923 'on another metric.')
2924 if not results_dict['confidence']:
2925 self.warnings.append('Confidence score is 0%. Try bisecting again on '
2926 'another platform or another metric.')
2927
2928 def FormatAndPrintResults(self, bisect_results):
2929 """Prints the results from a bisection run in a readable format.
2930
2931 Args:
2932 bisect_results: The results from a bisection test run.
2933 """
2934 revision_data = bisect_results['revision_data']
2935 revision_data_sorted = sorted(revision_data.iteritems(),
2936 key = lambda x: x[1]['sort'])
2937 results_dict = self._GetResultsDict(revision_data, revision_data_sorted)
2938
2939 self._CheckForWarnings(results_dict)
2940
2941 if self.opts.output_buildbot_annotations:
2942 bisect_utils.OutputAnnotationStepStart('Build Status Per Revision')
2943
2944 print
2945 print 'Full results of bisection:'
2946 for current_id, current_data in revision_data_sorted:
2947 build_status = current_data['passed']
2948
2949 if type(build_status) is bool:
2950 if build_status:
2951 build_status = 'Good'
2952 else:
2953 build_status = 'Bad'
2954
2955 print ' %20s %40s %s' % (current_data['depot'],
2956 current_id, build_status)
2957 print
2958
2959 if self.opts.output_buildbot_annotations:
2960 bisect_utils.OutputAnnotationStepClosed()
2961 # The perf dashboard scrapes the "results" step in order to comment on
2962 # bugs. If you change this, please update the perf dashboard as well.
2963 bisect_utils.OutputAnnotationStepStart('Results')
2964
2965 self._PrintBanner(results_dict)
2966 self._PrintWarnings()
2967
2968 if results_dict['culprit_revisions'] and results_dict['confidence']:
2969 for culprit in results_dict['culprit_revisions']:
2970 cl, info, depot = culprit
2971 self._PrintRevisionInfo(cl, info, depot)
2972 if results_dict['other_regressions']:
2973 self._PrintOtherRegressions(results_dict['other_regressions'],
2974 revision_data)
2975 self._PrintTestedCommitsTable(revision_data_sorted,
2976 results_dict['first_working_revision'],
2977 results_dict['last_broken_revision'],
2978 results_dict['confidence'])
2979 _PrintStepTime(revision_data_sorted)
2980 self._PrintReproSteps()
2981 _PrintThankYou()
2982 if self.opts.output_buildbot_annotations:
2983 bisect_utils.OutputAnnotationStepClosed()
2984
2985 def _PrintBanner(self, results_dict):
2986 if self._IsBisectModeReturnCode():
2987 metrics = 'N/A'
2988 change = 'Yes'
2989 else:
2990 metrics = '/'.join(self.opts.metric)
2991 change = '%.02f%% (+/-%.02f%%)' % (
2992 results_dict['regression_size'], results_dict['regression_std_err'])
2993
2994 if results_dict['culprit_revisions'] and results_dict['confidence']:
2995 status = self._ConfidenceLevelStatus(results_dict)
2996 else:
2997 status = 'Failure, could not reproduce.'
2998 change = 'Bisect could not reproduce a change.'
2999
3000 print RESULTS_BANNER % {
3001 'status': status,
3002 'command': self.opts.command,
3003 'metrics': metrics,
3004 'change': change,
3005 'confidence': results_dict['confidence'],
3006 }
3007
3008 def _PrintWarnings(self):
3009 """Prints a list of warning strings if there are any."""
3010 if not self.warnings:
3011 return
3012 print
3013 print 'WARNINGS:'
3014 for w in set(self.warnings):
3015 print ' ! %s' % w
3016
3017
3018 def _IsPlatformSupported():
3019 """Checks that this platform and build system are supported.
3020
3021 Args:
3022 opts: The options parsed from the command line.
3023
3024 Returns:
3025 True if the platform and build system are supported.
3026 """
3027 # Haven't tested the script out on any other platforms yet.
3028 supported = ['posix', 'nt']
3029 return os.name in supported
3030
3031
3032 def RmTreeAndMkDir(path_to_dir, skip_makedir=False):
3033 """Removes the directory tree specified, and then creates an empty
3034 directory in the same location (if not specified to skip).
3035
3036 Args:
3037 path_to_dir: Path to the directory tree.
3038 skip_makedir: Whether to skip creating empty directory, default is False.
3039
3040 Returns:
3041 True if successful, False if an error occurred.
3042 """
3043 try:
3044 if os.path.exists(path_to_dir):
3045 shutil.rmtree(path_to_dir)
3046 except OSError, e:
3047 if e.errno != errno.ENOENT:
3048 return False
3049
3050 if not skip_makedir:
3051 return MaybeMakeDirectory(path_to_dir)
3052
3053 return True
3054
3055
3056 def RemoveBuildFiles(build_type):
3057 """Removes build files from previous runs."""
3058 if RmTreeAndMkDir(os.path.join('out', build_type)):
3059 if RmTreeAndMkDir(os.path.join('build', build_type)):
3060 return True
3061 return False
3062
3063
3064 class BisectOptions(object):
3065 """Options to be used when running bisection."""
3066 def __init__(self):
3067 super(BisectOptions, self).__init__()
3068
3069 self.target_platform = 'chromium'
3070 self.build_preference = None
3071 self.good_revision = None
3072 self.bad_revision = None
3073 self.use_goma = None
3074 self.goma_dir = None
3075 self.cros_board = None
3076 self.cros_remote_ip = None
3077 self.repeat_test_count = 20
3078 self.truncate_percent = 25
3079 self.max_time_minutes = 20
3080 self.metric = None
3081 self.command = None
3082 self.output_buildbot_annotations = None
3083 self.no_custom_deps = False
3084 self.working_directory = None
3085 self.extra_src = None
3086 self.debug_ignore_build = None
3087 self.debug_ignore_sync = None
3088 self.debug_ignore_perf_test = None
3089 self.gs_bucket = None
3090 self.target_arch = 'ia32'
3091 self.target_build_type = 'Release'
3092 self.builder_host = None
3093 self.builder_port = None
3094 self.bisect_mode = BISECT_MODE_MEAN
3095
3096 @staticmethod
3097 def _CreateCommandLineParser():
3098 """Creates a parser with bisect options.
3099
3100 Returns:
3101 An instance of optparse.OptionParser.
3102 """
3103 usage = ('%prog [options] [-- chromium-options]\n'
3104 'Perform binary search on revision history to find a minimal '
3105 'range of revisions where a performance metric regressed.\n')
3106
3107 parser = optparse.OptionParser(usage=usage)
3108
3109 group = optparse.OptionGroup(parser, 'Bisect options')
3110 group.add_option('-c', '--command',
3111 type='str',
3112 help='A command to execute your performance test at' +
3113 ' each point in the bisection.')
3114 group.add_option('-b', '--bad_revision',
3115 type='str',
3116 help='A bad revision to start bisection. ' +
3117 'Must be later than good revision. May be either a git' +
3118 ' or svn revision.')
3119 group.add_option('-g', '--good_revision',
3120 type='str',
3121 help='A revision to start bisection where performance' +
3122 ' test is known to pass. Must be earlier than the ' +
3123 'bad revision. May be either a git or svn revision.')
3124 group.add_option('-m', '--metric',
3125 type='str',
3126 help='The desired metric to bisect on. For example ' +
3127 '"vm_rss_final_b/vm_rss_f_b"')
3128 group.add_option('-r', '--repeat_test_count',
3129 type='int',
3130 default=20,
3131 help='The number of times to repeat the performance '
3132 'test. Values will be clamped to range [1, 100]. '
3133 'Default value is 20.')
3134 group.add_option('--max_time_minutes',
3135 type='int',
3136 default=20,
3137 help='The maximum time (in minutes) to take running the '
3138 'performance tests. The script will run the performance '
3139 'tests according to --repeat_test_count, so long as it '
3140 'doesn\'t exceed --max_time_minutes. Values will be '
3141 'clamped to range [1, 60].'
3142 'Default value is 20.')
3143 group.add_option('-t', '--truncate_percent',
3144 type='int',
3145 default=25,
3146 help='The highest/lowest % are discarded to form a '
3147 'truncated mean. Values will be clamped to range [0, '
3148 '25]. Default value is 25 (highest/lowest 25% will be '
3149 'discarded).')
3150 group.add_option('--bisect_mode',
3151 type='choice',
3152 choices=[BISECT_MODE_MEAN, BISECT_MODE_STD_DEV,
3153 BISECT_MODE_RETURN_CODE],
3154 default=BISECT_MODE_MEAN,
3155 help='The bisect mode. Choices are to bisect on the '
3156 'difference in mean, std_dev, or return_code.')
3157 parser.add_option_group(group)
3158
3159 group = optparse.OptionGroup(parser, 'Build options')
3160 group.add_option('-w', '--working_directory',
3161 type='str',
3162 help='Path to the working directory where the script '
3163 'will do an initial checkout of the chromium depot. The '
3164 'files will be placed in a subdirectory "bisect" under '
3165 'working_directory and that will be used to perform the '
3166 'bisection. This parameter is optional, if it is not '
3167 'supplied, the script will work from the current depot.')
3168 group.add_option('--build_preference',
3169 type='choice',
3170 choices=['msvs', 'ninja', 'make'],
3171 help='The preferred build system to use. On linux/mac '
3172 'the options are make/ninja. On Windows, the options '
3173 'are msvs/ninja.')
3174 group.add_option('--target_platform',
3175 type='choice',
3176 choices=['chromium', 'cros', 'android', 'android-chrome'],
3177 default='chromium',
3178 help='The target platform. Choices are "chromium" '
3179 '(current platform), "cros", or "android". If you '
3180 'specify something other than "chromium", you must be '
3181 'properly set up to build that platform.')
3182 group.add_option('--no_custom_deps',
3183 dest='no_custom_deps',
3184 action='store_true',
3185 default=False,
3186 help='Run the script with custom_deps or not.')
3187 group.add_option('--extra_src',
3188 type='str',
3189 help='Path to a script which can be used to modify '
3190 'the bisect script\'s behavior.')
3191 group.add_option('--cros_board',
3192 type='str',
3193 help='The cros board type to build.')
3194 group.add_option('--cros_remote_ip',
3195 type='str',
3196 help='The remote machine to image to.')
3197 group.add_option('--use_goma',
3198 action='store_true',
3199 help='Add a bunch of extra threads for goma, and enable '
3200 'goma')
3201 group.add_option('--goma_dir',
3202 help='Path to goma tools (or system default if not '
3203 'specified).')
3204 group.add_option('--output_buildbot_annotations',
3205 action='store_true',
3206 help='Add extra annotation output for buildbot.')
3207 group.add_option('--gs_bucket',
3208 default='',
3209 dest='gs_bucket',
3210 type='str',
3211 help=('Name of Google Storage bucket to upload or '
3212 'download build. e.g., chrome-perf'))
3213 group.add_option('--target_arch',
3214 type='choice',
3215 choices=['ia32', 'x64', 'arm'],
3216 default='ia32',
3217 dest='target_arch',
3218 help=('The target build architecture. Choices are "ia32" '
3219 '(default), "x64" or "arm".'))
3220 group.add_option('--target_build_type',
3221 type='choice',
3222 choices=['Release', 'Debug'],
3223 default='Release',
3224 help='The target build type. Choices are "Release" '
3225 '(default), or "Debug".')
3226 group.add_option('--builder_host',
3227 dest='builder_host',
3228 type='str',
3229 help=('Host address of server to produce build by posting'
3230 ' try job request.'))
3231 group.add_option('--builder_port',
3232 dest='builder_port',
3233 type='int',
3234 help=('HTTP port of the server to produce build by posting'
3235 ' try job request.'))
3236 parser.add_option_group(group)
3237
3238 group = optparse.OptionGroup(parser, 'Debug options')
3239 group.add_option('--debug_ignore_build',
3240 action='store_true',
3241 help='DEBUG: Don\'t perform builds.')
3242 group.add_option('--debug_ignore_sync',
3243 action='store_true',
3244 help='DEBUG: Don\'t perform syncs.')
3245 group.add_option('--debug_ignore_perf_test',
3246 action='store_true',
3247 help='DEBUG: Don\'t perform performance tests.')
3248 parser.add_option_group(group)
3249 return parser
3250
3251 def ParseCommandLine(self):
3252 """Parses the command line for bisect options."""
3253 parser = self._CreateCommandLineParser()
3254 opts, _ = parser.parse_args()
3255
3256 try:
3257 if not opts.command:
3258 raise RuntimeError('missing required parameter: --command')
3259
3260 if not opts.good_revision:
3261 raise RuntimeError('missing required parameter: --good_revision')
3262
3263 if not opts.bad_revision:
3264 raise RuntimeError('missing required parameter: --bad_revision')
3265
3266 if not opts.metric and opts.bisect_mode != BISECT_MODE_RETURN_CODE:
3267 raise RuntimeError('missing required parameter: --metric')
3268
3269 if opts.gs_bucket:
3270 if not cloud_storage.List(opts.gs_bucket):
3271 raise RuntimeError('Invalid Google Storage: gs://%s' % opts.gs_bucket)
3272 if not opts.builder_host:
3273 raise RuntimeError('Must specify try server host name using '
3274 '--builder_host when gs_bucket is used.')
3275 if not opts.builder_port:
3276 raise RuntimeError('Must specify try server port number using '
3277 '--builder_port when gs_bucket is used.')
3278 if opts.target_platform == 'cros':
3279 # Run sudo up front to make sure credentials are cached for later.
3280 print 'Sudo is required to build cros:'
3281 print
3282 bisect_utils.RunProcess(['sudo', 'true'])
3283
3284 if not opts.cros_board:
3285 raise RuntimeError('missing required parameter: --cros_board')
3286
3287 if not opts.cros_remote_ip:
3288 raise RuntimeError('missing required parameter: --cros_remote_ip')
3289
3290 if not opts.working_directory:
3291 raise RuntimeError('missing required parameter: --working_directory')
3292
3293 if opts.bisect_mode != BISECT_MODE_RETURN_CODE:
3294 metric_values = opts.metric.split('/')
3295 if len(metric_values) != 2:
3296 raise RuntimeError('Invalid metric specified: [%s]' % opts.metric)
3297 opts.metric = metric_values
3298
3299 opts.repeat_test_count = min(max(opts.repeat_test_count, 1), 100)
3300 opts.max_time_minutes = min(max(opts.max_time_minutes, 1), 60)
3301 opts.truncate_percent = min(max(opts.truncate_percent, 0), 25)
3302 opts.truncate_percent = opts.truncate_percent / 100.0
3303
3304 for k, v in opts.__dict__.iteritems():
3305 assert hasattr(self, k), 'Invalid %s attribute in BisectOptions.' % k
3306 setattr(self, k, v)
3307 except RuntimeError, e:
3308 output_string = StringIO.StringIO()
3309 parser.print_help(file=output_string)
3310 error_message = '%s\n\n%s' % (e.message, output_string.getvalue())
3311 output_string.close()
3312 raise RuntimeError(error_message)
3313
3314 @staticmethod
3315 def FromDict(values):
3316 """Creates an instance of BisectOptions from a dictionary.
3317
3318 Args:
3319 values: a dict containing options to set.
3320
3321 Returns:
3322 An instance of BisectOptions.
3323 """
3324 opts = BisectOptions()
3325 for k, v in values.iteritems():
3326 assert hasattr(opts, k), 'Invalid %s attribute in BisectOptions.' % k
3327 setattr(opts, k, v)
3328
3329 if opts.metric and opts.bisect_mode != BISECT_MODE_RETURN_CODE:
3330 metric_values = opts.metric.split('/')
3331 if len(metric_values) != 2:
3332 raise RuntimeError('Invalid metric specified: [%s]' % opts.metric)
3333 opts.metric = metric_values
3334
3335 opts.repeat_test_count = min(max(opts.repeat_test_count, 1), 100)
3336 opts.max_time_minutes = min(max(opts.max_time_minutes, 1), 60)
3337 opts.truncate_percent = min(max(opts.truncate_percent, 0), 25)
3338 opts.truncate_percent = opts.truncate_percent / 100.0
3339
3340 return opts
3341
3342
3343 def main():
3344
3345 try:
3346 opts = BisectOptions()
3347 opts.ParseCommandLine()
3348
3349 if opts.extra_src:
3350 extra_src = bisect_utils.LoadExtraSrc(opts.extra_src)
3351 if not extra_src:
3352 raise RuntimeError('Invalid or missing --extra_src.')
3353 _AddAdditionalDepotInfo(extra_src.GetAdditionalDepotInfo())
3354
3355 if opts.working_directory:
3356 custom_deps = bisect_utils.DEFAULT_GCLIENT_CUSTOM_DEPS
3357 if opts.no_custom_deps:
3358 custom_deps = None
3359 bisect_utils.CreateBisectDirectoryAndSetupDepot(opts, custom_deps)
3360
3361 os.chdir(os.path.join(os.getcwd(), 'src'))
3362
3363 if not RemoveBuildFiles(opts.target_build_type):
3364 raise RuntimeError('Something went wrong removing the build files.')
3365
3366 if not _IsPlatformSupported():
3367 raise RuntimeError('Sorry, this platform isn\'t supported yet.')
3368
3369 # Check what source control method is being used, and create a
3370 # SourceControl object if possible.
3371 source_control = source_control_module.DetermineAndCreateSourceControl(opts)
3372
3373 if not source_control:
3374 raise RuntimeError(
3375 'Sorry, only the git workflow is supported at the moment.')
3376
3377 # gClient sync seems to fail if you're not in master branch.
3378 if (not source_control.IsInProperBranch() and
3379 not opts.debug_ignore_sync and
3380 not opts.working_directory):
3381 raise RuntimeError('You must switch to master branch to run bisection.')
3382 bisect_test = BisectPerformanceMetrics(source_control, opts)
3383 try:
3384 bisect_results = bisect_test.Run(opts.command,
3385 opts.bad_revision,
3386 opts.good_revision,
3387 opts.metric)
3388 if bisect_results['error']:
3389 raise RuntimeError(bisect_results['error'])
3390 bisect_test.FormatAndPrintResults(bisect_results)
3391 return 0
3392 finally:
3393 bisect_test.PerformCleanup()
3394 except RuntimeError, e:
3395 if opts.output_buildbot_annotations:
3396 # The perf dashboard scrapes the "results" step in order to comment on
3397 # bugs. If you change this, please update the perf dashboard as well.
3398 bisect_utils.OutputAnnotationStepStart('Results')
3399 print 'Error: %s' % e.message
3400 if opts.output_buildbot_annotations:
3401 bisect_utils.OutputAnnotationStepClosed()
3402 return 1
3403
3404
3405 if __name__ == '__main__':
3406 sys.exit(main())
OLDNEW
« no previous file with comments | « tools/auto_bisect/source_control.py ('k') | tools/bisect-perf-regression_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698