OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Performance Test Bisect Tool | 6 """Performance Test Bisect Tool |
7 | 7 |
8 This script bisects a series of changelists using binary search. It starts at | 8 This script bisects a series of changelists using binary search. It starts at |
9 a bad revision where a performance metric has regressed, and asks for a last | 9 a bad revision where a performance metric has regressed, and asks for a last |
10 known-good revision. It will then binary search across this revision range by | 10 known-good revision. It will then binary search across this revision range by |
(...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
145 # Android builds are also archived with the "full-build-linux prefix. | 145 # Android builds are also archived with the "full-build-linux prefix. |
146 return 'linux' | 146 return 'linux' |
147 if bisect_utils.IsMacHost(): | 147 if bisect_utils.IsMacHost(): |
148 return 'mac' | 148 return 'mac' |
149 raise NotImplementedError('Unknown platform "%s".' % sys.platform) | 149 raise NotImplementedError('Unknown platform "%s".' % sys.platform) |
150 | 150 |
151 base_name = 'full-build-%s' % PlatformName() | 151 base_name = 'full-build-%s' % PlatformName() |
152 if not build_revision: | 152 if not build_revision: |
153 return base_name | 153 return base_name |
154 if patch_sha: | 154 if patch_sha: |
155 build_revision = '%s_%s' % (build_revision , patch_sha) | 155 build_revision = '%s_%s' % (build_revision, patch_sha) |
156 return '%s_%s.zip' % (base_name, build_revision) | 156 return '%s_%s.zip' % (base_name, build_revision) |
157 | 157 |
158 | 158 |
159 def GetRemoteBuildPath(build_revision, target_platform='chromium', | 159 def GetRemoteBuildPath(build_revision, target_platform='chromium', |
160 target_arch='ia32', patch_sha=None): | 160 target_arch='ia32', patch_sha=None): |
161 """Returns the URL to download the build from.""" | 161 """Returns the URL to download the build from.""" |
162 def GetGSRootFolderName(target_platform): | 162 def GetGSRootFolderName(target_platform): |
163 """Returns the Google Cloud Storage root folder name.""" | 163 """Returns the Google Cloud Storage root folder name.""" |
164 if bisect_utils.IsWindowsHost(): | 164 if bisect_utils.IsWindowsHost(): |
165 if bisect_utils.Is64BitWindows() and target_arch == 'x64': | 165 if bisect_utils.Is64BitWindows() and target_arch == 'x64': |
(...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
266 os.chmod(os.path.join(output_dir, name), | 266 os.chmod(os.path.join(output_dir, name), |
267 zf.getinfo(name).external_attr >> 16L) | 267 zf.getinfo(name).external_attr >> 16L) |
268 | 268 |
269 | 269 |
270 def WriteStringToFile(text, file_name): | 270 def WriteStringToFile(text, file_name): |
271 """Writes text to a file, raising an RuntimeError on failure.""" | 271 """Writes text to a file, raising an RuntimeError on failure.""" |
272 try: | 272 try: |
273 with open(file_name, 'wb') as f: | 273 with open(file_name, 'wb') as f: |
274 f.write(text) | 274 f.write(text) |
275 except IOError: | 275 except IOError: |
276 raise RuntimeError('Error writing to file [%s]' % file_name ) | 276 raise RuntimeError('Error writing to file [%s]' % file_name) |
277 | 277 |
278 | 278 |
279 def ReadStringFromFile(file_name): | 279 def ReadStringFromFile(file_name): |
280 """Writes text to a file, raising an RuntimeError on failure.""" | 280 """Writes text to a file, raising an RuntimeError on failure.""" |
281 try: | 281 try: |
282 with open(file_name) as f: | 282 with open(file_name) as f: |
283 return f.read() | 283 return f.read() |
284 except IOError: | 284 except IOError: |
285 raise RuntimeError('Error reading file [%s]' % file_name ) | 285 raise RuntimeError('Error reading file [%s]' % file_name) |
286 | 286 |
287 | 287 |
288 def ChangeBackslashToSlashInPatch(diff_text): | 288 def ChangeBackslashToSlashInPatch(diff_text): |
289 """Formats file paths in the given patch text to Unix-style paths.""" | 289 """Formats file paths in the given patch text to Unix-style paths.""" |
290 if not diff_text: | 290 if not diff_text: |
291 return None | 291 return None |
292 diff_lines = diff_text.split('\n') | 292 diff_lines = diff_text.split('\n') |
293 for i in range(len(diff_lines)): | 293 for i in range(len(diff_lines)): |
294 line = diff_lines[i] | 294 line = diff_lines[i] |
295 if line.startswith('--- ') or line.startswith('+++ '): | 295 if line.startswith('--- ') or line.startswith('+++ '): |
(...skipping 14 matching lines...) Expand all Loading... |
310 rxp = re.compile('vars = {(?P<vars_body>[^}]+)', re.MULTILINE) | 310 rxp = re.compile('vars = {(?P<vars_body>[^}]+)', re.MULTILINE) |
311 re_results = rxp.search(deps_file_contents) | 311 re_results = rxp.search(deps_file_contents) |
312 | 312 |
313 if not re_results: | 313 if not re_results: |
314 return None | 314 return None |
315 | 315 |
316 # We should be left with a series of entries in the vars component of | 316 # We should be left with a series of entries in the vars component of |
317 # the DEPS file with the following format: | 317 # the DEPS file with the following format: |
318 # 'depot_name': 'revision', | 318 # 'depot_name': 'revision', |
319 vars_body = re_results.group('vars_body') | 319 vars_body = re_results.group('vars_body') |
320 rxp = re.compile("'(?P<depot_body>[\w_-]+)':[\s]+'(?P<rev_body>[\w@]+)'", | 320 rxp = re.compile(r"'(?P<depot_body>[\w_-]+)':[\s]+'(?P<rev_body>[\w@]+)'", |
321 re.MULTILINE) | 321 re.MULTILINE) |
322 re_results = rxp.findall(vars_body) | 322 re_results = rxp.findall(vars_body) |
323 | 323 |
324 return dict(re_results) | 324 return dict(re_results) |
325 | 325 |
326 | 326 |
327 def _WaitUntilBuildIsReady( | 327 def _WaitUntilBuildIsReady( |
328 fetch_build, bot_name, builder_host, builder_port, build_request_id, | 328 fetch_build, bot_name, builder_host, builder_port, build_request_id, |
329 max_timeout): | 329 max_timeout): |
330 """Waits until build is produced by bisect builder on try server. | 330 """Waits until build is produced by bisect builder on try server. |
(...skipping 165 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
496 | 496 |
497 Returns: | 497 Returns: |
498 A list of floating point numbers found. | 498 A list of floating point numbers found. |
499 """ | 499 """ |
500 # Format is: RESULT <graph>: <trace>= <value> <units> | 500 # Format is: RESULT <graph>: <trace>= <value> <units> |
501 metric_re = re.escape('RESULT %s: %s=' % (metric[0], metric[1])) | 501 metric_re = re.escape('RESULT %s: %s=' % (metric[0], metric[1])) |
502 | 502 |
503 # The log will be parsed looking for format: | 503 # The log will be parsed looking for format: |
504 # <*>RESULT <graph_name>: <trace_name>= <value> | 504 # <*>RESULT <graph_name>: <trace_name>= <value> |
505 single_result_re = re.compile( | 505 single_result_re = re.compile( |
506 metric_re + '\s*(?P<VALUE>[-]?\d*(\.\d*)?)') | 506 metric_re + r'\s*(?P<VALUE>[-]?\d*(\.\d*)?)') |
507 | 507 |
508 # The log will be parsed looking for format: | 508 # The log will be parsed looking for format: |
509 # <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...] | 509 # <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...] |
510 multi_results_re = re.compile( | 510 multi_results_re = re.compile( |
511 metric_re + '\s*\[\s*(?P<VALUES>[-]?[\d\., ]+)\s*\]') | 511 metric_re + r'\s*\[\s*(?P<VALUES>[-]?[\d\., ]+)\s*\]') |
512 | 512 |
513 # The log will be parsed looking for format: | 513 # The log will be parsed looking for format: |
514 # <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} | 514 # <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} |
515 mean_stddev_re = re.compile( | 515 mean_stddev_re = re.compile( |
516 metric_re + | 516 metric_re + |
517 '\s*\{\s*(?P<MEAN>[-]?\d*(\.\d*)?),\s*(?P<STDDEV>\d+(\.\d*)?)\s*\}') | 517 r'\s*\{\s*(?P<MEAN>[-]?\d*(\.\d*)?),\s*(?P<STDDEV>\d+(\.\d*)?)\s*\}') |
518 | 518 |
519 text_lines = text.split('\n') | 519 text_lines = text.split('\n') |
520 values_list = [] | 520 values_list = [] |
521 for current_line in text_lines: | 521 for current_line in text_lines: |
522 # Parse the output from the performance test for the metric we're | 522 # Parse the output from the performance test for the metric we're |
523 # interested in. | 523 # interested in. |
524 single_result_match = single_result_re.search(current_line) | 524 single_result_match = single_result_re.search(current_line) |
525 multi_results_match = multi_results_re.search(current_line) | 525 multi_results_match = multi_results_re.search(current_line) |
526 mean_stddev_match = mean_stddev_re.search(current_line) | 526 mean_stddev_match = mean_stddev_re.search(current_line) |
527 if (not single_result_match is None and | 527 if (not single_result_match is None and |
(...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
696 raise RunGitError('Must be in a git repository to send changes to trybots.') | 696 raise RunGitError('Must be in a git repository to send changes to trybots.') |
697 | 697 |
698 current_branch = current_branch.strip() | 698 current_branch = current_branch.strip() |
699 # Make sure current branch is master. | 699 # Make sure current branch is master. |
700 if current_branch != parent_branch: | 700 if current_branch != parent_branch: |
701 output, returncode = bisect_utils.RunGit(['checkout', '-f', parent_branch]) | 701 output, returncode = bisect_utils.RunGit(['checkout', '-f', parent_branch]) |
702 if returncode: | 702 if returncode: |
703 raise RunGitError('Failed to checkout branch: %s.' % output) | 703 raise RunGitError('Failed to checkout branch: %s.' % output) |
704 | 704 |
705 # Delete new branch if exists. | 705 # Delete new branch if exists. |
706 output, returncode = bisect_utils.RunGit(['branch', '--list' ]) | 706 output, returncode = bisect_utils.RunGit(['branch', '--list']) |
707 if new_branch in output: | 707 if new_branch in output: |
708 output, returncode = bisect_utils.RunGit(['branch', '-D', new_branch]) | 708 output, returncode = bisect_utils.RunGit(['branch', '-D', new_branch]) |
709 if returncode: | 709 if returncode: |
710 raise RunGitError('Deleting branch failed, %s', output) | 710 raise RunGitError('Deleting branch failed, %s', output) |
711 | 711 |
712 # Check if the tree is dirty: make sure the index is up to date and then | 712 # Check if the tree is dirty: make sure the index is up to date and then |
713 # run diff-index. | 713 # run diff-index. |
714 bisect_utils.RunGit(['update-index', '--refresh', '-q']) | 714 bisect_utils.RunGit(['update-index', '--refresh', '-q']) |
715 output, returncode = bisect_utils.RunGit(['diff-index', 'HEAD']) | 715 output, returncode = bisect_utils.RunGit(['diff-index', 'HEAD']) |
716 if output: | 716 if output: |
(...skipping 146 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
863 execfile(deps_file, {}, deps_data) | 863 execfile(deps_file, {}, deps_data) |
864 deps_data = deps_data['deps'] | 864 deps_data = deps_data['deps'] |
865 | 865 |
866 rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)") | 866 rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)") |
867 results = {} | 867 results = {} |
868 for depot_name, depot_data in bisect_utils.DEPOT_DEPS_NAME.iteritems(): | 868 for depot_name, depot_data in bisect_utils.DEPOT_DEPS_NAME.iteritems(): |
869 if (depot_data.get('platform') and | 869 if (depot_data.get('platform') and |
870 depot_data.get('platform') != os.name): | 870 depot_data.get('platform') != os.name): |
871 continue | 871 continue |
872 | 872 |
873 if (depot_data.get('recurse') and depot in depot_data.get('from')): | 873 if depot_data.get('recurse') and depot in depot_data.get('from'): |
874 depot_data_src = depot_data.get('src') or depot_data.get('src_old') | 874 depot_data_src = depot_data.get('src') or depot_data.get('src_old') |
875 src_dir = deps_data.get(depot_data_src) | 875 src_dir = deps_data.get(depot_data_src) |
876 if src_dir: | 876 if src_dir: |
877 self.depot_registry.SetDepotDir(depot_name, os.path.join( | 877 self.depot_registry.SetDepotDir(depot_name, os.path.join( |
878 self.src_cwd, depot_data_src[4:])) | 878 self.src_cwd, depot_data_src[4:])) |
879 re_results = rxp.search(src_dir) | 879 re_results = rxp.search(src_dir) |
880 if re_results: | 880 if re_results: |
881 results[depot_name] = re_results.group('revision') | 881 results[depot_name] = re_results.group('revision') |
882 else: | 882 else: |
883 warning_text = ('Could not parse revision for %s while bisecting ' | 883 warning_text = ('Could not parse revision for %s while bisecting ' |
(...skipping 522 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1406 """ | 1406 """ |
1407 if self.opts.target_platform in ['android']: | 1407 if self.opts.target_platform in ['android']: |
1408 # When its a third_party depot, get the chromium revision. | 1408 # When its a third_party depot, get the chromium revision. |
1409 if depot != 'chromium': | 1409 if depot != 'chromium': |
1410 revision = bisect_utils.CheckRunGit( | 1410 revision = bisect_utils.CheckRunGit( |
1411 ['rev-parse', 'HEAD'], cwd=self.src_cwd).strip() | 1411 ['rev-parse', 'HEAD'], cwd=self.src_cwd).strip() |
1412 commit_position = source_control.GetCommitPosition(revision, | 1412 commit_position = source_control.GetCommitPosition(revision, |
1413 cwd=self.src_cwd) | 1413 cwd=self.src_cwd) |
1414 if not commit_position: | 1414 if not commit_position: |
1415 return command_to_run | 1415 return command_to_run |
1416 cmd_re = re.compile('--browser=(?P<browser_type>\S+)') | 1416 cmd_re = re.compile(r'--browser=(?P<browser_type>\S+)') |
1417 matches = cmd_re.search(command_to_run) | 1417 matches = cmd_re.search(command_to_run) |
1418 if bisect_utils.IsStringInt(commit_position) and matches: | 1418 if bisect_utils.IsStringInt(commit_position) and matches: |
1419 cmd_browser = matches.group('browser_type') | 1419 cmd_browser = matches.group('browser_type') |
1420 if commit_position <= 274857 and cmd_browser == 'android-chrome-shell': | 1420 if commit_position <= 274857 and cmd_browser == 'android-chrome-shell': |
1421 return command_to_run.replace(cmd_browser, | 1421 return command_to_run.replace(cmd_browser, |
1422 'android-chromium-testshell') | 1422 'android-chromium-testshell') |
1423 elif (commit_position >= 276628 and | 1423 elif (commit_position >= 276628 and |
1424 cmd_browser == 'android-chromium-testshell'): | 1424 cmd_browser == 'android-chromium-testshell'): |
1425 return command_to_run.replace(cmd_browser, | 1425 return command_to_run.replace(cmd_browser, |
1426 'android-chrome-shell') | 1426 'android-chrome-shell') |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1495 current_args.append('--reset-results') | 1495 current_args.append('--reset-results') |
1496 elif i == self.opts.repeat_test_count - 1 and upload_on_last_run: | 1496 elif i == self.opts.repeat_test_count - 1 and upload_on_last_run: |
1497 current_args.append('--upload-results') | 1497 current_args.append('--upload-results') |
1498 if results_label: | 1498 if results_label: |
1499 current_args.append('--results-label=%s' % results_label) | 1499 current_args.append('--results-label=%s' % results_label) |
1500 try: | 1500 try: |
1501 output, return_code = bisect_utils.RunProcessAndRetrieveOutput( | 1501 output, return_code = bisect_utils.RunProcessAndRetrieveOutput( |
1502 current_args, cwd=self.src_cwd) | 1502 current_args, cwd=self.src_cwd) |
1503 except OSError, e: | 1503 except OSError, e: |
1504 if e.errno == errno.ENOENT: | 1504 if e.errno == errno.ENOENT: |
1505 err_text = ('Something went wrong running the performance test. ' | 1505 err_text = ('Something went wrong running the performance test. ' |
1506 'Please review the command line:\n\n') | 1506 'Please review the command line:\n\n') |
1507 if 'src/' in ' '.join(args): | 1507 if 'src/' in ' '.join(args): |
1508 err_text += ('Check that you haven\'t accidentally specified a ' | 1508 err_text += ('Check that you haven\'t accidentally specified a ' |
1509 'path with src/ in the command.\n\n') | 1509 'path with src/ in the command.\n\n') |
1510 err_text += ' '.join(args) | 1510 err_text += ' '.join(args) |
1511 err_text += '\n' | 1511 err_text += '\n' |
1512 | 1512 |
1513 return (err_text, failure_code) | 1513 return (err_text, failure_code) |
1514 raise | 1514 raise |
1515 | 1515 |
1516 output_of_all_runs += output | 1516 output_of_all_runs += output |
(...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1612 commit_position, d, bisect_utils.DEPOT_DEPS_NAME, -1000) | 1612 commit_position, d, bisect_utils.DEPOT_DEPS_NAME, -1000) |
1613 | 1613 |
1614 if dependant_rev: | 1614 if dependant_rev: |
1615 revisions_to_sync.append([d, dependant_rev]) | 1615 revisions_to_sync.append([d, dependant_rev]) |
1616 | 1616 |
1617 num_resolved = len(revisions_to_sync) | 1617 num_resolved = len(revisions_to_sync) |
1618 num_needed = len(bisect_utils.DEPOT_DEPS_NAME[depot]['depends']) | 1618 num_needed = len(bisect_utils.DEPOT_DEPS_NAME[depot]['depends']) |
1619 | 1619 |
1620 self.depot_registry.ChangeToDepotDir(depot) | 1620 self.depot_registry.ChangeToDepotDir(depot) |
1621 | 1621 |
1622 if not ((num_resolved - 1) == num_needed): | 1622 if not (num_resolved - 1) == num_needed: |
1623 return None | 1623 return None |
1624 | 1624 |
1625 return revisions_to_sync | 1625 return revisions_to_sync |
1626 | 1626 |
1627 def PerformPreBuildCleanup(self): | 1627 def PerformPreBuildCleanup(self): |
1628 """Performs cleanup between runs.""" | 1628 """Performs cleanup between runs.""" |
1629 print 'Cleaning up between runs.' | 1629 print 'Cleaning up between runs.' |
1630 print | 1630 print |
1631 | 1631 |
1632 # Leaving these .pyc files around between runs may disrupt some perf tests. | 1632 # Leaving these .pyc files around between runs may disrupt some perf tests. |
(...skipping 1311 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2944 bisect_utils.OutputAnnotationStepStart('Results') | 2944 bisect_utils.OutputAnnotationStepStart('Results') |
2945 print 'Error: ', e.message | 2945 print 'Error: ', e.message |
2946 logging.warn('A RuntimeError was caught: %s', e.message) | 2946 logging.warn('A RuntimeError was caught: %s', e.message) |
2947 if opts.output_buildbot_annotations: | 2947 if opts.output_buildbot_annotations: |
2948 bisect_utils.OutputAnnotationStepClosed() | 2948 bisect_utils.OutputAnnotationStepClosed() |
2949 return 1 | 2949 return 1 |
2950 | 2950 |
2951 | 2951 |
2952 if __name__ == '__main__': | 2952 if __name__ == '__main__': |
2953 sys.exit(main()) | 2953 sys.exit(main()) |
OLD | NEW |