| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Performance Test Bisect Tool | 6 """Performance Test Bisect Tool |
| 7 | 7 |
| 8 This script bisects a series of changelists using binary search. It starts at | 8 This script bisects a series of changelists using binary search. It starts at |
| 9 a bad revision where a performance metric has regressed, and asks for a last | 9 a bad revision where a performance metric has regressed, and asks for a last |
| 10 known-good revision. It will then binary search across this revision range by | 10 known-good revision. It will then binary search across this revision range by |
| (...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 145 # Android builds are also archived with the "full-build-linux prefix. | 145 # Android builds are also archived with the "full-build-linux prefix. |
| 146 return 'linux' | 146 return 'linux' |
| 147 if bisect_utils.IsMacHost(): | 147 if bisect_utils.IsMacHost(): |
| 148 return 'mac' | 148 return 'mac' |
| 149 raise NotImplementedError('Unknown platform "%s".' % sys.platform) | 149 raise NotImplementedError('Unknown platform "%s".' % sys.platform) |
| 150 | 150 |
| 151 base_name = 'full-build-%s' % PlatformName() | 151 base_name = 'full-build-%s' % PlatformName() |
| 152 if not build_revision: | 152 if not build_revision: |
| 153 return base_name | 153 return base_name |
| 154 if patch_sha: | 154 if patch_sha: |
| 155 build_revision = '%s_%s' % (build_revision , patch_sha) | 155 build_revision = '%s_%s' % (build_revision, patch_sha) |
| 156 return '%s_%s.zip' % (base_name, build_revision) | 156 return '%s_%s.zip' % (base_name, build_revision) |
| 157 | 157 |
| 158 | 158 |
| 159 def GetRemoteBuildPath(build_revision, target_platform='chromium', | 159 def GetRemoteBuildPath(build_revision, target_platform='chromium', |
| 160 target_arch='ia32', patch_sha=None): | 160 target_arch='ia32', patch_sha=None): |
| 161 """Returns the URL to download the build from.""" | 161 """Returns the URL to download the build from.""" |
| 162 def GetGSRootFolderName(target_platform): | 162 def GetGSRootFolderName(target_platform): |
| 163 """Returns the Google Cloud Storage root folder name.""" | 163 """Returns the Google Cloud Storage root folder name.""" |
| 164 if bisect_utils.IsWindowsHost(): | 164 if bisect_utils.IsWindowsHost(): |
| 165 if bisect_utils.Is64BitWindows() and target_arch == 'x64': | 165 if bisect_utils.Is64BitWindows() and target_arch == 'x64': |
| (...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 266 os.chmod(os.path.join(output_dir, name), | 266 os.chmod(os.path.join(output_dir, name), |
| 267 zf.getinfo(name).external_attr >> 16L) | 267 zf.getinfo(name).external_attr >> 16L) |
| 268 | 268 |
| 269 | 269 |
| 270 def WriteStringToFile(text, file_name): | 270 def WriteStringToFile(text, file_name): |
| 271 """Writes text to a file, raising an RuntimeError on failure.""" | 271 """Writes text to a file, raising an RuntimeError on failure.""" |
| 272 try: | 272 try: |
| 273 with open(file_name, 'wb') as f: | 273 with open(file_name, 'wb') as f: |
| 274 f.write(text) | 274 f.write(text) |
| 275 except IOError: | 275 except IOError: |
| 276 raise RuntimeError('Error writing to file [%s]' % file_name ) | 276 raise RuntimeError('Error writing to file [%s]' % file_name) |
| 277 | 277 |
| 278 | 278 |
| 279 def ReadStringFromFile(file_name): | 279 def ReadStringFromFile(file_name): |
| 280 """Writes text to a file, raising an RuntimeError on failure.""" | 280 """Writes text to a file, raising an RuntimeError on failure.""" |
| 281 try: | 281 try: |
| 282 with open(file_name) as f: | 282 with open(file_name) as f: |
| 283 return f.read() | 283 return f.read() |
| 284 except IOError: | 284 except IOError: |
| 285 raise RuntimeError('Error reading file [%s]' % file_name ) | 285 raise RuntimeError('Error reading file [%s]' % file_name) |
| 286 | 286 |
| 287 | 287 |
| 288 def ChangeBackslashToSlashInPatch(diff_text): | 288 def ChangeBackslashToSlashInPatch(diff_text): |
| 289 """Formats file paths in the given patch text to Unix-style paths.""" | 289 """Formats file paths in the given patch text to Unix-style paths.""" |
| 290 if not diff_text: | 290 if not diff_text: |
| 291 return None | 291 return None |
| 292 diff_lines = diff_text.split('\n') | 292 diff_lines = diff_text.split('\n') |
| 293 for i in range(len(diff_lines)): | 293 for i in range(len(diff_lines)): |
| 294 line = diff_lines[i] | 294 line = diff_lines[i] |
| 295 if line.startswith('--- ') or line.startswith('+++ '): | 295 if line.startswith('--- ') or line.startswith('+++ '): |
| (...skipping 14 matching lines...) Expand all Loading... |
| 310 rxp = re.compile('vars = {(?P<vars_body>[^}]+)', re.MULTILINE) | 310 rxp = re.compile('vars = {(?P<vars_body>[^}]+)', re.MULTILINE) |
| 311 re_results = rxp.search(deps_file_contents) | 311 re_results = rxp.search(deps_file_contents) |
| 312 | 312 |
| 313 if not re_results: | 313 if not re_results: |
| 314 return None | 314 return None |
| 315 | 315 |
| 316 # We should be left with a series of entries in the vars component of | 316 # We should be left with a series of entries in the vars component of |
| 317 # the DEPS file with the following format: | 317 # the DEPS file with the following format: |
| 318 # 'depot_name': 'revision', | 318 # 'depot_name': 'revision', |
| 319 vars_body = re_results.group('vars_body') | 319 vars_body = re_results.group('vars_body') |
| 320 rxp = re.compile("'(?P<depot_body>[\w_-]+)':[\s]+'(?P<rev_body>[\w@]+)'", | 320 rxp = re.compile(r"'(?P<depot_body>[\w_-]+)':[\s]+'(?P<rev_body>[\w@]+)'", |
| 321 re.MULTILINE) | 321 re.MULTILINE) |
| 322 re_results = rxp.findall(vars_body) | 322 re_results = rxp.findall(vars_body) |
| 323 | 323 |
| 324 return dict(re_results) | 324 return dict(re_results) |
| 325 | 325 |
| 326 | 326 |
| 327 def _WaitUntilBuildIsReady( | 327 def _WaitUntilBuildIsReady( |
| 328 fetch_build, bot_name, builder_host, builder_port, build_request_id, | 328 fetch_build, bot_name, builder_host, builder_port, build_request_id, |
| 329 max_timeout): | 329 max_timeout): |
| 330 """Waits until build is produced by bisect builder on try server. | 330 """Waits until build is produced by bisect builder on try server. |
| (...skipping 165 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 496 | 496 |
| 497 Returns: | 497 Returns: |
| 498 A list of floating point numbers found. | 498 A list of floating point numbers found. |
| 499 """ | 499 """ |
| 500 # Format is: RESULT <graph>: <trace>= <value> <units> | 500 # Format is: RESULT <graph>: <trace>= <value> <units> |
| 501 metric_re = re.escape('RESULT %s: %s=' % (metric[0], metric[1])) | 501 metric_re = re.escape('RESULT %s: %s=' % (metric[0], metric[1])) |
| 502 | 502 |
| 503 # The log will be parsed looking for format: | 503 # The log will be parsed looking for format: |
| 504 # <*>RESULT <graph_name>: <trace_name>= <value> | 504 # <*>RESULT <graph_name>: <trace_name>= <value> |
| 505 single_result_re = re.compile( | 505 single_result_re = re.compile( |
| 506 metric_re + '\s*(?P<VALUE>[-]?\d*(\.\d*)?)') | 506 metric_re + r'\s*(?P<VALUE>[-]?\d*(\.\d*)?)') |
| 507 | 507 |
| 508 # The log will be parsed looking for format: | 508 # The log will be parsed looking for format: |
| 509 # <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...] | 509 # <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...] |
| 510 multi_results_re = re.compile( | 510 multi_results_re = re.compile( |
| 511 metric_re + '\s*\[\s*(?P<VALUES>[-]?[\d\., ]+)\s*\]') | 511 metric_re + r'\s*\[\s*(?P<VALUES>[-]?[\d\., ]+)\s*\]') |
| 512 | 512 |
| 513 # The log will be parsed looking for format: | 513 # The log will be parsed looking for format: |
| 514 # <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} | 514 # <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} |
| 515 mean_stddev_re = re.compile( | 515 mean_stddev_re = re.compile( |
| 516 metric_re + | 516 metric_re + |
| 517 '\s*\{\s*(?P<MEAN>[-]?\d*(\.\d*)?),\s*(?P<STDDEV>\d+(\.\d*)?)\s*\}') | 517 r'\s*\{\s*(?P<MEAN>[-]?\d*(\.\d*)?),\s*(?P<STDDEV>\d+(\.\d*)?)\s*\}') |
| 518 | 518 |
| 519 text_lines = text.split('\n') | 519 text_lines = text.split('\n') |
| 520 values_list = [] | 520 values_list = [] |
| 521 for current_line in text_lines: | 521 for current_line in text_lines: |
| 522 # Parse the output from the performance test for the metric we're | 522 # Parse the output from the performance test for the metric we're |
| 523 # interested in. | 523 # interested in. |
| 524 single_result_match = single_result_re.search(current_line) | 524 single_result_match = single_result_re.search(current_line) |
| 525 multi_results_match = multi_results_re.search(current_line) | 525 multi_results_match = multi_results_re.search(current_line) |
| 526 mean_stddev_match = mean_stddev_re.search(current_line) | 526 mean_stddev_match = mean_stddev_re.search(current_line) |
| 527 if (not single_result_match is None and | 527 if (not single_result_match is None and |
| (...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 696 raise RunGitError('Must be in a git repository to send changes to trybots.') | 696 raise RunGitError('Must be in a git repository to send changes to trybots.') |
| 697 | 697 |
| 698 current_branch = current_branch.strip() | 698 current_branch = current_branch.strip() |
| 699 # Make sure current branch is master. | 699 # Make sure current branch is master. |
| 700 if current_branch != parent_branch: | 700 if current_branch != parent_branch: |
| 701 output, returncode = bisect_utils.RunGit(['checkout', '-f', parent_branch]) | 701 output, returncode = bisect_utils.RunGit(['checkout', '-f', parent_branch]) |
| 702 if returncode: | 702 if returncode: |
| 703 raise RunGitError('Failed to checkout branch: %s.' % output) | 703 raise RunGitError('Failed to checkout branch: %s.' % output) |
| 704 | 704 |
| 705 # Delete new branch if exists. | 705 # Delete new branch if exists. |
| 706 output, returncode = bisect_utils.RunGit(['branch', '--list' ]) | 706 output, returncode = bisect_utils.RunGit(['branch', '--list']) |
| 707 if new_branch in output: | 707 if new_branch in output: |
| 708 output, returncode = bisect_utils.RunGit(['branch', '-D', new_branch]) | 708 output, returncode = bisect_utils.RunGit(['branch', '-D', new_branch]) |
| 709 if returncode: | 709 if returncode: |
| 710 raise RunGitError('Deleting branch failed, %s', output) | 710 raise RunGitError('Deleting branch failed, %s', output) |
| 711 | 711 |
| 712 # Check if the tree is dirty: make sure the index is up to date and then | 712 # Check if the tree is dirty: make sure the index is up to date and then |
| 713 # run diff-index. | 713 # run diff-index. |
| 714 bisect_utils.RunGit(['update-index', '--refresh', '-q']) | 714 bisect_utils.RunGit(['update-index', '--refresh', '-q']) |
| 715 output, returncode = bisect_utils.RunGit(['diff-index', 'HEAD']) | 715 output, returncode = bisect_utils.RunGit(['diff-index', 'HEAD']) |
| 716 if output: | 716 if output: |
| (...skipping 146 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 863 execfile(deps_file, {}, deps_data) | 863 execfile(deps_file, {}, deps_data) |
| 864 deps_data = deps_data['deps'] | 864 deps_data = deps_data['deps'] |
| 865 | 865 |
| 866 rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)") | 866 rxp = re.compile(".git@(?P<revision>[a-fA-F0-9]+)") |
| 867 results = {} | 867 results = {} |
| 868 for depot_name, depot_data in bisect_utils.DEPOT_DEPS_NAME.iteritems(): | 868 for depot_name, depot_data in bisect_utils.DEPOT_DEPS_NAME.iteritems(): |
| 869 if (depot_data.get('platform') and | 869 if (depot_data.get('platform') and |
| 870 depot_data.get('platform') != os.name): | 870 depot_data.get('platform') != os.name): |
| 871 continue | 871 continue |
| 872 | 872 |
| 873 if (depot_data.get('recurse') and depot in depot_data.get('from')): | 873 if depot_data.get('recurse') and depot in depot_data.get('from'): |
| 874 depot_data_src = depot_data.get('src') or depot_data.get('src_old') | 874 depot_data_src = depot_data.get('src') or depot_data.get('src_old') |
| 875 src_dir = deps_data.get(depot_data_src) | 875 src_dir = deps_data.get(depot_data_src) |
| 876 if src_dir: | 876 if src_dir: |
| 877 self.depot_registry.SetDepotDir(depot_name, os.path.join( | 877 self.depot_registry.SetDepotDir(depot_name, os.path.join( |
| 878 self.src_cwd, depot_data_src[4:])) | 878 self.src_cwd, depot_data_src[4:])) |
| 879 re_results = rxp.search(src_dir) | 879 re_results = rxp.search(src_dir) |
| 880 if re_results: | 880 if re_results: |
| 881 results[depot_name] = re_results.group('revision') | 881 results[depot_name] = re_results.group('revision') |
| 882 else: | 882 else: |
| 883 warning_text = ('Could not parse revision for %s while bisecting ' | 883 warning_text = ('Could not parse revision for %s while bisecting ' |
| (...skipping 522 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1406 """ | 1406 """ |
| 1407 if self.opts.target_platform in ['android']: | 1407 if self.opts.target_platform in ['android']: |
| 1408 # When its a third_party depot, get the chromium revision. | 1408 # When its a third_party depot, get the chromium revision. |
| 1409 if depot != 'chromium': | 1409 if depot != 'chromium': |
| 1410 revision = bisect_utils.CheckRunGit( | 1410 revision = bisect_utils.CheckRunGit( |
| 1411 ['rev-parse', 'HEAD'], cwd=self.src_cwd).strip() | 1411 ['rev-parse', 'HEAD'], cwd=self.src_cwd).strip() |
| 1412 commit_position = source_control.GetCommitPosition(revision, | 1412 commit_position = source_control.GetCommitPosition(revision, |
| 1413 cwd=self.src_cwd) | 1413 cwd=self.src_cwd) |
| 1414 if not commit_position: | 1414 if not commit_position: |
| 1415 return command_to_run | 1415 return command_to_run |
| 1416 cmd_re = re.compile('--browser=(?P<browser_type>\S+)') | 1416 cmd_re = re.compile(r'--browser=(?P<browser_type>\S+)') |
| 1417 matches = cmd_re.search(command_to_run) | 1417 matches = cmd_re.search(command_to_run) |
| 1418 if bisect_utils.IsStringInt(commit_position) and matches: | 1418 if bisect_utils.IsStringInt(commit_position) and matches: |
| 1419 cmd_browser = matches.group('browser_type') | 1419 cmd_browser = matches.group('browser_type') |
| 1420 if commit_position <= 274857 and cmd_browser == 'android-chrome-shell': | 1420 if commit_position <= 274857 and cmd_browser == 'android-chrome-shell': |
| 1421 return command_to_run.replace(cmd_browser, | 1421 return command_to_run.replace(cmd_browser, |
| 1422 'android-chromium-testshell') | 1422 'android-chromium-testshell') |
| 1423 elif (commit_position >= 276628 and | 1423 elif (commit_position >= 276628 and |
| 1424 cmd_browser == 'android-chromium-testshell'): | 1424 cmd_browser == 'android-chromium-testshell'): |
| 1425 return command_to_run.replace(cmd_browser, | 1425 return command_to_run.replace(cmd_browser, |
| 1426 'android-chrome-shell') | 1426 'android-chrome-shell') |
| (...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1495 current_args.append('--reset-results') | 1495 current_args.append('--reset-results') |
| 1496 elif i == self.opts.repeat_test_count - 1 and upload_on_last_run: | 1496 elif i == self.opts.repeat_test_count - 1 and upload_on_last_run: |
| 1497 current_args.append('--upload-results') | 1497 current_args.append('--upload-results') |
| 1498 if results_label: | 1498 if results_label: |
| 1499 current_args.append('--results-label=%s' % results_label) | 1499 current_args.append('--results-label=%s' % results_label) |
| 1500 try: | 1500 try: |
| 1501 output, return_code = bisect_utils.RunProcessAndRetrieveOutput( | 1501 output, return_code = bisect_utils.RunProcessAndRetrieveOutput( |
| 1502 current_args, cwd=self.src_cwd) | 1502 current_args, cwd=self.src_cwd) |
| 1503 except OSError, e: | 1503 except OSError, e: |
| 1504 if e.errno == errno.ENOENT: | 1504 if e.errno == errno.ENOENT: |
| 1505 err_text = ('Something went wrong running the performance test. ' | 1505 err_text = ('Something went wrong running the performance test. ' |
| 1506 'Please review the command line:\n\n') | 1506 'Please review the command line:\n\n') |
| 1507 if 'src/' in ' '.join(args): | 1507 if 'src/' in ' '.join(args): |
| 1508 err_text += ('Check that you haven\'t accidentally specified a ' | 1508 err_text += ('Check that you haven\'t accidentally specified a ' |
| 1509 'path with src/ in the command.\n\n') | 1509 'path with src/ in the command.\n\n') |
| 1510 err_text += ' '.join(args) | 1510 err_text += ' '.join(args) |
| 1511 err_text += '\n' | 1511 err_text += '\n' |
| 1512 | 1512 |
| 1513 return (err_text, failure_code) | 1513 return (err_text, failure_code) |
| 1514 raise | 1514 raise |
| 1515 | 1515 |
| 1516 output_of_all_runs += output | 1516 output_of_all_runs += output |
| (...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1612 commit_position, d, bisect_utils.DEPOT_DEPS_NAME, -1000) | 1612 commit_position, d, bisect_utils.DEPOT_DEPS_NAME, -1000) |
| 1613 | 1613 |
| 1614 if dependant_rev: | 1614 if dependant_rev: |
| 1615 revisions_to_sync.append([d, dependant_rev]) | 1615 revisions_to_sync.append([d, dependant_rev]) |
| 1616 | 1616 |
| 1617 num_resolved = len(revisions_to_sync) | 1617 num_resolved = len(revisions_to_sync) |
| 1618 num_needed = len(bisect_utils.DEPOT_DEPS_NAME[depot]['depends']) | 1618 num_needed = len(bisect_utils.DEPOT_DEPS_NAME[depot]['depends']) |
| 1619 | 1619 |
| 1620 self.depot_registry.ChangeToDepotDir(depot) | 1620 self.depot_registry.ChangeToDepotDir(depot) |
| 1621 | 1621 |
| 1622 if not ((num_resolved - 1) == num_needed): | 1622 if not (num_resolved - 1) == num_needed: |
| 1623 return None | 1623 return None |
| 1624 | 1624 |
| 1625 return revisions_to_sync | 1625 return revisions_to_sync |
| 1626 | 1626 |
| 1627 def PerformPreBuildCleanup(self): | 1627 def PerformPreBuildCleanup(self): |
| 1628 """Performs cleanup between runs.""" | 1628 """Performs cleanup between runs.""" |
| 1629 print 'Cleaning up between runs.' | 1629 print 'Cleaning up between runs.' |
| 1630 print | 1630 print |
| 1631 | 1631 |
| 1632 # Leaving these .pyc files around between runs may disrupt some perf tests. | 1632 # Leaving these .pyc files around between runs may disrupt some perf tests. |
| (...skipping 1311 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2944 bisect_utils.OutputAnnotationStepStart('Results') | 2944 bisect_utils.OutputAnnotationStepStart('Results') |
| 2945 print 'Error: ', e.message | 2945 print 'Error: ', e.message |
| 2946 logging.warn('A RuntimeError was caught: %s', e.message) | 2946 logging.warn('A RuntimeError was caught: %s', e.message) |
| 2947 if opts.output_buildbot_annotations: | 2947 if opts.output_buildbot_annotations: |
| 2948 bisect_utils.OutputAnnotationStepClosed() | 2948 bisect_utils.OutputAnnotationStepClosed() |
| 2949 return 1 | 2949 return 1 |
| 2950 | 2950 |
| 2951 | 2951 |
| 2952 if __name__ == '__main__': | 2952 if __name__ == '__main__': |
| 2953 sys.exit(main()) | 2953 sys.exit(main()) |
| OLD | NEW |