| OLD | NEW |
| 1 # Copyright (c) 2010 Google Inc. All rights reserved. | 1 # Copyright (c) 2010 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 317 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 328 self._builder_data = {} | 328 self._builder_data = {} |
| 329 | 329 |
| 330 def builder_data(self): | 330 def builder_data(self): |
| 331 if not self._builder_data: | 331 if not self._builder_data: |
| 332 for builder_name in self._release_builders(): | 332 for builder_name in self._release_builders(): |
| 333 builder = self._tool.buildbot.builder_with_name(builder_name) | 333 builder = self._tool.buildbot.builder_with_name(builder_name) |
| 334 builder_results = builder.latest_layout_test_results() | 334 builder_results = builder.latest_layout_test_results() |
| 335 if builder_results: | 335 if builder_results: |
| 336 self._builder_data[builder_name] = builder_results | 336 self._builder_data[builder_name] = builder_results |
| 337 else: | 337 else: |
| 338 _log.warning("No result for builder '%s'" % builder_name) | 338 raise Exception("No result for builder %s." % builder_name) |
| 339 return self._builder_data | 339 return self._builder_data |
| 340 | 340 |
| 341 # The release builders cycle much faster than the debug ones and cover all t
he platforms. | 341 # The release builders cycle much faster than the debug ones and cover all t
he platforms. |
| 342 def _release_builders(self): | 342 def _release_builders(self): |
| 343 release_builders = [] | 343 release_builders = [] |
| 344 for builder_name in builders.all_builder_names(): | 344 for builder_name in builders.all_builder_names(): |
| 345 if builder_name.find('ASAN') != -1: | 345 if builder_name.find('ASAN') != -1: |
| 346 continue | 346 continue |
| 347 port = self._tool.port_factory.get_from_builder_name(builder_name) | 347 port = self._tool.port_factory.get_from_builder_name(builder_name) |
| 348 if port.test_configuration().build_type == 'release': | 348 if port.test_configuration().build_type == 'release': |
| (...skipping 332 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 681 ]) | 681 ]) |
| 682 | 682 |
| 683 def bot_revision_data(self): | 683 def bot_revision_data(self): |
| 684 revisions = [] | 684 revisions = [] |
| 685 for result in self.builder_data().values(): | 685 for result in self.builder_data().values(): |
| 686 if result.run_was_interrupted(): | 686 if result.run_was_interrupted(): |
| 687 _log.error("Can't rebaseline because the latest run on %s exited
early." % result.builder_name()) | 687 _log.error("Can't rebaseline because the latest run on %s exited
early." % result.builder_name()) |
| 688 return [] | 688 return [] |
| 689 revisions.append({ | 689 revisions.append({ |
| 690 "builder": result.builder_name(), | 690 "builder": result.builder_name(), |
| 691 "revision": result.blink_revision(), | 691 "revision": result.chromium_revision(), |
| 692 }) | 692 }) |
| 693 return revisions | 693 return revisions |
| 694 | 694 |
| 695 def tests_to_rebaseline(self, tool, min_revision, print_revisions): | 695 def tests_to_rebaseline(self, tool, min_revision, print_revisions): |
| 696 port = tool.port_factory.get() | 696 port = tool.port_factory.get() |
| 697 expectations_file_path = port.path_to_generic_test_expectations_file() | 697 expectations_file_path = port.path_to_generic_test_expectations_file() |
| 698 | 698 |
| 699 tests = set() | 699 tests = set() |
| 700 revision = None | 700 revision = None |
| 701 commit = None |
| 701 author = None | 702 author = None |
| 702 bugs = set() | 703 bugs = set() |
| 703 has_any_needs_rebaseline_lines = False | 704 has_any_needs_rebaseline_lines = False |
| 704 | 705 |
| 705 for line in tool.scm().blame(expectations_file_path).split("\n"): | 706 for line in tool.scm().blame(expectations_file_path).split("\n"): |
| 706 comment_index = line.find("#") | 707 comment_index = line.find("#") |
| 707 if comment_index == -1: | 708 if comment_index == -1: |
| 708 comment_index = len(line) | 709 comment_index = len(line) |
| 709 line_without_comments = re.sub(r"\s+", " ", line[:comment_index].str
ip()) | 710 line_without_comments = re.sub(r"\s+", " ", line[:comment_index].str
ip()) |
| 710 | 711 |
| 711 if "NeedsRebaseline" not in line_without_comments: | 712 if "NeedsRebaseline" not in line_without_comments: |
| 712 continue | 713 continue |
| 713 | 714 |
| 714 has_any_needs_rebaseline_lines = True | 715 has_any_needs_rebaseline_lines = True |
| 715 | 716 |
| 716 parsed_line = re.match("^(\S*)[^(]*\((\S*).*?([^ ]*)\ \[[^[]*$", lin
e_without_comments) | 717 parsed_line = re.match("^(\S*)[^(]*\((\S*).*?([^ ]*)\ \[[^[]*$", lin
e_without_comments) |
| 717 | 718 |
| 718 commit_hash = parsed_line.group(1) | 719 commit_hash = parsed_line.group(1) |
| 719 svn_revision = tool.scm().svn_revision_from_git_commit(commit_hash) | 720 chromium_revision = tool.scm().chromium_revision_from_git_commit(com
mit_hash) |
| 720 | 721 |
| 721 test = parsed_line.group(3) | 722 test = parsed_line.group(3) |
| 722 if print_revisions: | 723 if print_revisions: |
| 723 _log.info("%s is waiting for r%s" % (test, svn_revision)) | 724 _log.info("%s is waiting for r%s" % (test, chromium_revision)) |
| 724 | 725 |
| 725 if not svn_revision or svn_revision > min_revision: | 726 if not chromium_revision or chromium_revision > min_revision: |
| 726 continue | 727 continue |
| 727 | 728 |
| 728 if revision and svn_revision != revision: | 729 if revision and chromium_revision != revision: |
| 729 continue | 730 continue |
| 730 | 731 |
| 731 if not revision: | 732 if not revision: |
| 732 revision = svn_revision | 733 revision = chromium_revision |
| 734 commit = commit_hash |
| 733 author = parsed_line.group(2) | 735 author = parsed_line.group(2) |
| 734 | 736 |
| 735 bugs.update(re.findall("crbug\.com\/(\d+)", line_without_comments)) | 737 bugs.update(re.findall("crbug\.com\/(\d+)", line_without_comments)) |
| 736 tests.add(test) | 738 tests.add(test) |
| 737 | 739 |
| 738 if len(tests) >= self.MAX_LINES_TO_REBASELINE: | 740 if len(tests) >= self.MAX_LINES_TO_REBASELINE: |
| 739 _log.info("Too many tests to rebaseline in one patch. Doing the
first %d." % self.MAX_LINES_TO_REBASELINE) | 741 _log.info("Too many tests to rebaseline in one patch. Doing the
first %d." % self.MAX_LINES_TO_REBASELINE) |
| 740 break | 742 break |
| 741 | 743 |
| 742 return tests, revision, author, bugs, has_any_needs_rebaseline_lines | 744 return tests, revision, commit, author, bugs, has_any_needs_rebaseline_l
ines |
| 743 | 745 |
| 744 def link_to_patch(self, revision): | 746 def link_to_patch(self, commit): |
| 745 return "http://src.chromium.org/viewvc/blink?view=revision&revision=" +
str(revision) | 747 return "https://chromium.googlesource.com/chromium/src/+/" + commit |
| 746 | 748 |
| 747 def commit_message(self, author, revision, bugs): | 749 def commit_message(self, author, revision, commit, bugs): |
| 748 bug_string = "" | 750 bug_string = "" |
| 749 if bugs: | 751 if bugs: |
| 750 bug_string = "BUG=%s\n" % ",".join(bugs) | 752 bug_string = "BUG=%s\n" % ",".join(bugs) |
| 751 | 753 |
| 752 return """Auto-rebaseline for r%s | 754 return """Auto-rebaseline for r%s |
| 753 | 755 |
| 754 %s | 756 %s |
| 755 | 757 |
| 756 %sTBR=%s | 758 %sTBR=%s |
| 757 """ % (revision, self.link_to_patch(revision), bug_string, author) | 759 """ % (revision, self.link_to_patch(commit), bug_string, author) |
| 758 | 760 |
| 759 def get_test_prefix_list(self, tests): | 761 def get_test_prefix_list(self, tests): |
| 760 test_prefix_list = {} | 762 test_prefix_list = {} |
| 761 lines_to_remove = {} | 763 lines_to_remove = {} |
| 762 | 764 |
| 763 for builder_name in self._release_builders(): | 765 for builder_name in self._release_builders(): |
| 764 port_name = builders.port_name_for_builder_name(builder_name) | 766 port_name = builders.port_name_for_builder_name(builder_name) |
| 765 port = self._tool.port_factory.get(port_name) | 767 port = self._tool.port_factory.get(port_name) |
| 766 expectations = TestExpectations(port, include_overrides=True) | 768 expectations = TestExpectations(port, include_overrides=True) |
| 767 for test in expectations.get_needs_rebaseline_failures(): | 769 for test in expectations.get_needs_rebaseline_failures(): |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 816 | 818 |
| 817 if tool.scm().has_working_directory_changes(): | 819 if tool.scm().has_working_directory_changes(): |
| 818 _log.error("Cannot proceed with working directory changes. Clean wor
king directory first.") | 820 _log.error("Cannot proceed with working directory changes. Clean wor
king directory first.") |
| 819 return | 821 return |
| 820 | 822 |
| 821 revision_data = self.bot_revision_data() | 823 revision_data = self.bot_revision_data() |
| 822 if not revision_data: | 824 if not revision_data: |
| 823 return | 825 return |
| 824 | 826 |
| 825 min_revision = int(min([item["revision"] for item in revision_data])) | 827 min_revision = int(min([item["revision"] for item in revision_data])) |
| 826 tests, revision, author, bugs, has_any_needs_rebaseline_lines = self.tes
ts_to_rebaseline(tool, min_revision, print_revisions=options.verbose) | 828 tests, revision, commit, author, bugs, has_any_needs_rebaseline_lines =
self.tests_to_rebaseline(tool, min_revision, print_revisions=options.verbose) |
| 827 | 829 |
| 828 if options.verbose: | 830 if options.verbose: |
| 829 _log.info("Min revision across all bots is %s." % min_revision) | 831 _log.info("Min revision across all bots is %s." % min_revision) |
| 830 for item in revision_data: | 832 for item in revision_data: |
| 831 _log.info("%s: r%s" % (item["builder"], item["revision"])) | 833 _log.info("%s: r%s" % (item["builder"], item["revision"])) |
| 832 | 834 |
| 833 if not tests: | 835 if not tests: |
| 834 _log.debug('No tests to rebaseline.') | 836 _log.debug('No tests to rebaseline.') |
| 835 return | 837 return |
| 836 | 838 |
| 837 if self.tree_status() == 'closed': | 839 if self.tree_status() == 'closed': |
| 838 _log.info('Cannot proceed. Tree is closed.') | 840 _log.info('Cannot proceed. Tree is closed.') |
| 839 return | 841 return |
| 840 | 842 |
| 841 _log.info('Rebaselining %s for r%s by %s.' % (list(tests), revision, aut
hor)) | 843 _log.info('Rebaselining %s for r%s by %s.' % (list(tests), revision, aut
hor)) |
| 842 | 844 |
| 843 test_prefix_list, lines_to_remove = self.get_test_prefix_list(tests) | 845 test_prefix_list, lines_to_remove = self.get_test_prefix_list(tests) |
| 844 | 846 |
| 845 did_finish = False | 847 did_finish = False |
| 846 old_branch_name_or_ref = '' | 848 old_branch_name_or_ref = '' |
| 847 rebaseline_branch_name = self.AUTO_REBASELINE_BRANCH_NAME | 849 rebaseline_branch_name = self.AUTO_REBASELINE_BRANCH_NAME |
| 848 try: | 850 try: |
| 849 # Setup git-svn for dcommit if necessary. | |
| 850 if tool.executive.run_command( | |
| 851 ['git', 'config', '--local', '--get-regexp', r'^svn-remote\.
'], | |
| 852 return_exit_code=True): | |
| 853 tool.executive.run_command(['git', 'auto-svn']) | |
| 854 | |
| 855 # Save the current branch name and checkout a clean branch for the p
atch. | 851 # Save the current branch name and checkout a clean branch for the p
atch. |
| 856 old_branch_name_or_ref = _get_branch_name_or_ref(tool) | 852 old_branch_name_or_ref = _get_branch_name_or_ref(tool) |
| 857 if old_branch_name_or_ref == self.AUTO_REBASELINE_BRANCH_NAME: | 853 if old_branch_name_or_ref == self.AUTO_REBASELINE_BRANCH_NAME: |
| 858 rebaseline_branch_name = self.AUTO_REBASELINE_ALT_BRANCH_NAME | 854 rebaseline_branch_name = self.AUTO_REBASELINE_ALT_BRANCH_NAME |
| 859 tool.scm().delete_branch(rebaseline_branch_name) | 855 tool.scm().delete_branch(rebaseline_branch_name) |
| 860 tool.scm().create_clean_branch(rebaseline_branch_name) | 856 tool.scm().create_clean_branch(rebaseline_branch_name) |
| 861 | 857 |
| 862 # If the tests are passing everywhere, then this list will be empty.
We don't need | 858 # If the tests are passing everywhere, then this list will be empty.
We don't need |
| 863 # to rebaseline, but we'll still need to update TestExpectations. | 859 # to rebaseline, but we'll still need to update TestExpectations. |
| 864 if test_prefix_list: | 860 if test_prefix_list: |
| 865 self._rebaseline(options, test_prefix_list) | 861 self._rebaseline(options, test_prefix_list) |
| 866 | 862 |
| 867 tool.scm().commit_locally_with_message(self.commit_message(author, r
evision, bugs)) | 863 tool.scm().commit_locally_with_message(self.commit_message(author, r
evision, commit, bugs)) |
| 868 | 864 |
| 869 # FIXME: It would be nice if we could dcommit the patch without uplo
ading, but still | 865 # FIXME: It would be nice if we could dcommit the patch without uplo
ading, but still |
| 870 # go through all the precommit hooks. For rebaselines with lots of f
iles, uploading | 866 # go through all the precommit hooks. For rebaselines with lots of f
iles, uploading |
| 871 # takes a long time and sometimes fails, but we don't want to commit
if, e.g. the | 867 # takes a long time and sometimes fails, but we don't want to commit
if, e.g. the |
| 872 # tree is closed. | 868 # tree is closed. |
| 873 did_finish = self._run_git_cl_command(options, ['upload', '-f']) | 869 did_finish = self._run_git_cl_command(options, ['upload', '-f']) |
| 874 | 870 |
| 875 if did_finish: | 871 if did_finish: |
| 876 # Uploading can take a very long time. Do another pull to make s
ure TestExpectations is up to date, | 872 # Uploading can take a very long time. Do another pull to make s
ure TestExpectations is up to date, |
| 877 # so the dcommit can go through. | 873 # so the dcommit can go through. |
| (...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 969 self._tool.scm().checkout_branch(old_branch_name_or_ref) | 965 self._tool.scm().checkout_branch(old_branch_name_or_ref) |
| 970 else: | 966 else: |
| 971 self._log_queue.put(self.QUIT_LOG) | 967 self._log_queue.put(self.QUIT_LOG) |
| 972 log_thread.join() | 968 log_thread.join() |
| 973 | 969 |
| 974 def execute(self, options, args, tool): | 970 def execute(self, options, args, tool): |
| 975 self._verbose = options.verbose | 971 self._verbose = options.verbose |
| 976 while True: | 972 while True: |
| 977 self._do_one_rebaseline() | 973 self._do_one_rebaseline() |
| 978 time.sleep(self.SLEEP_TIME_IN_SECONDS) | 974 time.sleep(self.SLEEP_TIME_IN_SECONDS) |
| OLD | NEW |